diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 6d00f78456..b4be024437 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1891,7 +1891,11 @@ run_lm_eval() { export INFERENCEX_LM_EVAL_RUNTIME_READY=true fi - local openai_server_base="http://0.0.0.0:${port}" + # Most launchers run eval beside the API process and keep the historical + # 0.0.0.0 default. Orchestrators such as srt-slurm can place the benchmark + # client on a different node, so allow them to provide the routed host. + local openai_server_host="${EVAL_SERVER_HOST:-0.0.0.0}" + local openai_server_base="http://${openai_server_host}:${port}" local openai_chat_base="${openai_server_base}/v1/chat/completions" export OPENAI_API_KEY=${OPENAI_API_KEY:-EMPTY} MODEL_NAME=${MODEL_NAME:-$MODEL} # Prefer MODEL_NAME, else MODEL diff --git a/benchmarks/multi_node/agentic_srt.sh b/benchmarks/multi_node/agentic_srt.sh index abaf15a974..91278cbba3 100644 --- a/benchmarks/multi_node/agentic_srt.sh +++ b/benchmarks/multi_node/agentic_srt.sh @@ -60,16 +60,86 @@ if [[ "${EVAL_ONLY:-false}" == "true" ]]; then _wait_for_openai_chat_route --port "$PORT" fi +# Preserve the legacy DP-attention replay contract. The SGLang router uses this +# header to keep every request in one AgentX correlation tree on a stable DP +# route, which is important for both session continuity and prefix-cache reuse. +if [[ "${PREFILL_DP_ATTN:-false}" == "true" ]]; then + export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true +fi + +# Reset every advertised SGLang worker before each concurrency point, matching +# the retired amd_utils trace replay. /flush_cache covers GPU radix + host +# HiCache; the storage-backend endpoint is best-effort because L3 is optional. +clear_agentic_worker_caches() { + local timeout_seconds="${FLUSH_DRAIN_TIMEOUT:-120}" + local metrics_csv="${AIPERF_SERVER_METRICS_URLS:-}" + if [[ -z "$metrics_csv" ]]; then + if [[ "${AIPERF_DRAIN_BACKEND:-dynamo}" == "sglang" ]]; then + echo "[clear_caches] ERROR: native SGLang requires all worker metrics URLs" >&2 + return 1 + fi + echo "[clear_caches] WARN: AIPERF_SERVER_METRICS_URLS unset; skipping cache flush" >&2 + return 0 + fi + + local -a metrics_urls + if [[ "${AIPERF_DRAIN_BACKEND:-dynamo}" == "sglang" && \ + ( "$metrics_csv" == ,* || "$metrics_csv" == *, || "$metrics_csv" == *,,* ) ]]; then + echo "[clear_caches] ERROR: empty worker metrics URL" >&2 + return 1 + fi + IFS=',' read -r -a metrics_urls <<< "$metrics_csv" + local metrics_url base_url start response code flushed + for metrics_url in "${metrics_urls[@]}"; do + [[ -n "$metrics_url" ]] || continue + base_url="${metrics_url%/metrics}" + start=$(date +%s) + flushed=0 + response="" + while :; do + response=$(curl -sf -m 10 -X POST "${base_url}/flush_cache" 2>/dev/null || true) + if grep -qi "Cache flushed" <<< "$response"; then + flushed=1 + break + fi + if (( $(date +%s) - start >= timeout_seconds )); then + break + fi + sleep 3 + done + if (( flushed )); then + echo "[clear_caches] ${base_url}: L1+L2 flushed" + else + if [[ "${AIPERF_DRAIN_BACKEND:-dynamo}" == "sglang" ]]; then + echo "[clear_caches] ERROR ${base_url}: L1+L2 flush not confirmed after ${timeout_seconds}s" >&2 + return 1 + fi + echo "[clear_caches] WARN ${base_url}: L1+L2 flush not confirmed after ${timeout_seconds}s" >&2 + fi + + code=$(curl -s -m 60 -o /dev/null -w '%{http_code}' \ + -X POST "${base_url}/hicache/storage-backend/clear" 2>/dev/null || true) + if [[ "$code" == "200" ]]; then + echo "[clear_caches] ${base_url}: L3 store cleared" + else + echo "[clear_caches] ${base_url}: L3 clear http=${code:-000} (optional backend unavailable)" + fi + done +} + wait_for_agentic_servers_idle() { local timeout_seconds="${AIPERF_DRAIN_TIMEOUT_SECONDS:-1800}" local poll_seconds="${AIPERF_DRAIN_POLL_SECONDS:-10}" - local frontend_metrics_url="${AIPERF_SERVER_URL%/}/metrics" + local frontend_metrics_url="${AIPERF_FRONTEND_METRICS_URL:-${AIPERF_SERVER_URL:-http://${SRT_FRONTEND_HOST:-localhost}:${PORT}}/metrics}" "$AIPERF_PYTHON" - \ "$timeout_seconds" \ "$poll_seconds" \ "$frontend_metrics_url" \ - "${AIPERF_SERVER_METRICS_URLS:-}" <<'PY' + "${AIPERF_SERVER_METRICS_URLS:-}" \ + "${AIPERF_DRAIN_BACKEND:-dynamo}" \ + "${DISAGG:-false}" <<'PY' +import math import sys import time import urllib.request @@ -77,7 +147,15 @@ import urllib.request timeout_seconds = int(sys.argv[1]) poll_seconds = int(sys.argv[2]) frontend_url = sys.argv[3] -worker_urls = [url for url in sys.argv[4].split(",") if url] +worker_urls = [url.strip() for url in sys.argv[4].split(",")] +backend = sys.argv[5] +disagg = sys.argv[6].lower() == "true" +if backend not in {"sglang", "dynamo"}: + raise SystemExit(f"Unsupported AIPERF_DRAIN_BACKEND: {backend}") +if not all(worker_urls): + raise SystemExit("Agentic drain requires non-empty worker metrics URLs") +if timeout_seconds <= 0 or poll_seconds <= 0: + raise SystemExit("Agentic drain timeout and poll interval must be positive") deadline = time.monotonic() + timeout_seconds idle_polls = 0 @@ -87,32 +165,67 @@ def fetch_metrics(url: str) -> str: return response.read().decode("utf-8") -def metric_sum(metrics: str, name: str) -> float: - total = 0.0 +def metric_sum(metrics: str, name: str, *, required: bool = True) -> float: + values = [] for line in metrics.splitlines(): if not line or line.startswith("#"): continue fields = line.split() if len(fields) < 2 or fields[0].split("{", 1)[0] != name: continue - total += float(fields[1]) - return total + value = float(fields[1]) + if not math.isfinite(value) or value < 0: + raise ValueError(f"Invalid request gauge {name}: {value}") + values.append(value) + if required and not values: + raise ValueError(f"Missing required request gauge: {name}") + return sum(values) + + +def worker_requests(metrics: str) -> float: + # SGLang's scheduler also holds requests outside running/waiting while KV + # is bootstrapping or transferring. These queues must drain before flush. + # Names match sglang/srt/observability/metrics_collector.py. + is_sglang = backend == "sglang" or "sglang:num_running_reqs" in metrics + if not is_sglang: + if "trtllm_num_requests_running" in metrics: + return metric_sum(metrics, "trtllm_num_requests_running") + metric_sum( + metrics, "trtllm_num_requests_waiting" + ) + return metric_sum(metrics, "vllm:num_requests_running") + metric_sum( + metrics, "vllm:num_requests_waiting" + ) + active = metric_sum(metrics, "sglang:num_running_reqs") + metric_sum( + metrics, "sglang:num_queue_reqs" + ) + active += metric_sum(metrics, "sglang:num_grammar_queue_reqs", required=False) + for name in ( + "sglang:num_prefill_bootstrap_queue_reqs", + "sglang:num_prefill_inflight_queue_reqs", + "sglang:num_decode_prealloc_queue_reqs", + "sglang:num_decode_transfer_queue_reqs", + ): + active += metric_sum(metrics, name, required=disagg) + return active while time.monotonic() < deadline: try: - frontend_metrics = fetch_metrics(frontend_url) - frontend_active = metric_sum(frontend_metrics, "dynamo_frontend_active_requests") + frontend_active = 0.0 + # Native SGLang's router does not expose Dynamo's active-request gauge; + # its worker schedulers (including all PD queues) are authoritative. + if backend == "dynamo": + frontend_active = metric_sum( + fetch_metrics(frontend_url), "dynamo_frontend_active_requests" + ) worker_active = 0.0 for worker_url in worker_urls: worker_metrics = fetch_metrics(worker_url) - worker_active += metric_sum(worker_metrics, "vllm:num_requests_running") - worker_active += metric_sum(worker_metrics, "vllm:num_requests_waiting") - worker_active += metric_sum(worker_metrics, "trtllm_num_requests_running") - worker_active += metric_sum(worker_metrics, "trtllm_num_requests_waiting") + worker_active += worker_requests(worker_metrics) + frontend_status = f"{frontend_active:g}" if backend == "dynamo" else "n/a" print( - f"Agentic drain status: frontend_active={frontend_active:g} " - f"worker_running_or_waiting={worker_active:g}", + f"Agentic drain status: backend={backend} frontend_active={frontend_status} " + f"worker_pending_requests={worker_active:g}", flush=True, ) if frontend_active == 0 and worker_active == 0: @@ -145,6 +258,9 @@ for index in "${!CONCURRENCIES[@]}"; do mkdir -p "$RESULT_DIR" echo "Running agentic concurrency $concurrency of: ${CONCURRENCIES[*]}" + if [[ "${CLEAR_CACHE_BETWEEN_CONC:-0}" == "1" ]]; then + clear_agentic_worker_caches + fi build_replay_cmd "$RESULT_DIR" run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi300x-amd.yaml b/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi300x-amd.yaml new file mode 100644 index 0000000000..65596e12e1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi300x-amd.yaml @@ -0,0 +1,24 @@ +# MI300X AMD fleet: shared NFS source/results, node-local images and HF cache. +# Slurm selects available nodes from compute; drained nodes remain excluded. +cluster: mi300x-amd +default_partition: compute +default_time_limit: "04:00:00" +gpus_per_node: 8 +accelerator_vendor: amd +network_interface: ens50f1np1 +gpu_sbatch_directive: gres +use_segment_sbatch_directive: false +use_exclusive_sbatch_directive: true +runtime_config_transport: shared-filesystem +record_launch_plan: true +default_sbatch_directives: + # Full-node allocation: all 128 logical CPUs and all RAM on this fleet. + cpus-per-task: "128" + mem: "0" +default_bash_preamble: "ulimit -l unlimited; ulimit -c 0" +default_mounts: + /dev/kfd: /dev/kfd + /dev/dri: /dev/dri + /dev/infiniband: /dev/infiniband + /raid/inferencex/models: /hf-cache +nginx_raise_ulimit: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi300x/agentic/disagg-1p1d-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi300x/agentic/disagg-1p1d-dep8-mtp.yaml new file mode 100644 index 0000000000..691ac2f78b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi300x/agentic/disagg-1p1d-dep8-mtp.yaml @@ -0,0 +1,200 @@ +# AgentX Qwen3.5 FP8: DEP8 prefill and DEP8 decode, native MORI IO/EP. +# Both roles have attention TP=1; keep Mamba/GQA transfer layouts homogeneous. +# Source-audited candidate, not yet runtime-validated. +# Uses the srt-slurm AMD runtime and native Qwen3.5 MTP. +# Concurrency candidates are untuned; fast runs are not performance submissions. +# The shared launcher supplies the cluster profile and workspace/results mounts. +# srt-slurm and Pyxis own model caching and the container lifecycle. + +name: mi300x-qwen35-agentx-mori-disagg-1p1d-dep8-mtp + +model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910 + precision: fp8 + +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + container: + image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910@sha256:985488daa99010f6c72f17286b05804165e26c683be932325487d10b4ea6fd49 + +slurm: + time_limit: "24:00:00" + +resources: + gpu_type: mi300x + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + prefill-policy: cache_aware + decode-policy: round_robin + cache-threshold: 0.3 + dp-aware: true + health-check-timeout-secs: 600 + health-check-interval-secs: 30 + +backend: + type: sglang + prefill_environment: &worker_environment + HF_HOME: /hf-cache + HF_HUB_CACHE: /hf-cache/hub + HUGGINGFACE_HUB_CACHE: /hf-cache/hub + PYTHONDONTWRITEBYTECODE: "1" + PYTHONUNBUFFERED: "1" + PYTHONNOUSERSITE: "1" + GLOO_SOCKET_IFNAME: ens50f1np1 + NCCL_SOCKET_IFNAME: ens50f1np1 + MORI_SOCKET_IFNAME: ens50f1np1 + MORI_RDMA_DEVICES: bnxt_re_bond0,bnxt_re_bond1,bnxt_re_bond2,bnxt_re_bond3,bnxt_re_bond4,bnxt_re_bond5,bnxt_re_bond6,bnxt_re_bond7 + IBDEVICES: bnxt_re_bond0,bnxt_re_bond1,bnxt_re_bond2,bnxt_re_bond3,bnxt_re_bond4,bnxt_re_bond5,bnxt_re_bond6,bnxt_re_bond7 + MORI_IB_GID_INDEX: "3" + MORI_RDMA_TC: "104" + MORI_RDMA_SL: "3" + MORI_IO_TC: "104" + MORI_IO_SL: "3" + NCCL_IB_TC: "104" + NCCL_IB_SL: "3" + NCCL_IB_GID_INDEX: "3" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000" + MORI_IO_QP_MAX_SEND_WR: "16384" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_SHMEM_MODE: ISOLATION + # SGLang divides the 8192-token prefill chunk by DP8 before MoRI setup. + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" + # Cache-aware routing can select a rank other than bootstrap_room % DP. + SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK: "1" + SGLANG_USE_AITER: "1" + SGLANG_USE_AITER_UNIFIED_ATTN: "1" + AITER_FLYDSL_FORCE: "1" + SGLANG_MAMBA_SSM_DTYPE: bfloat16 + SGLANG_MORI_QP_PER_TRANSFER: "4" + SGLANG_MORI_NUM_WORKERS: "4" + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_HEALTH_CHECK_TIMEOUT: "600" + SGLANG_TIMEOUT_KEEP_ALIVE: "1800" + decode_environment: + <<: *worker_environment + # 128 global requests / DP8 * 4 MTP tokens = 64 tokens per rank. + # Keep dispatch capacity above the admission-derived token bound. + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + # AgentX throughput uses the existing Qwen3.5 native-MTP golden target. + # Retained for every AgentX run, including fast bring-up; evals disable them. + SGLANG_SIMULATE_ACC_LEN: "3.39" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + sglang_config: + prefill: &worker_config + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + dp-size: 8 + enable-dp-attention: true + expert-parallel-size: 8 + moe-a2a-backend: mori + deepep-mode: normal + disable-shared-experts-fusion: true + disaggregation-transfer-backend: mori + disaggregation-ib-device: bnxt_re_bond0,bnxt_re_bond1,bnxt_re_bond2,bnxt_re_bond3,bnxt_re_bond4,bnxt_re_bond5,bnxt_re_bond6,bnxt_re_bond7 + dtype: bfloat16 + # auto follows the explicit BF16 model activation dtype, not FP8 weights. + kv-cache-dtype: auto + attention-backend: aiter + # Image enables INT8 quick-reduce; do not combine with allreduce fusion. + enable-aiter-allreduce-fusion: false + page-size: 16 + mamba-ssm-dtype: bfloat16 + moe-dense-tp-size: 1 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + context-length: 262144 + disable-radix-cache: false + mem-fraction-static: 0.75 + max-running-requests: 128 + max-prefill-tokens: 32768 + chunked-prefill-size: 8192 + scheduler-recv-interval: 30 + tokenizer-worker-num: 6 + cuda-graph-max-bs-decode: 64 + watchdog-timeout: 3600 + enable-metrics: true + enable-cache-report: true + log-level: info + decode: + <<: *worker_config + tensor-parallel-size: 8 + expert-parallel-size: 8 + dp-size: 8 + +srun_options: + container-writable: "" + container-remap-root: "" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +output: + record_launch_plan: true + +benchmark: + type: custom + command: | + set -euo pipefail + export PORT="${SRT_FRONTEND_PORT}" + export AIPERF_SERVER_URL="http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" + export RESULT_DIR="/results/${SLURM_JOB_ID}/agentic" + export AGENTIC_OUTPUT_DIR="/results/${SLURM_JOB_ID}" + mkdir -p "$RESULT_DIR" "$AGENTIC_OUTPUT_DIR" + exec bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + MODEL_PREFIX: qwen3.5 + FRAMEWORK: sglang-disagg + PRECISION: fp8 + CONC: "32" + CONC_LIST: "32 64" + DURATION: "3600" + RESULT_FILENAME: qwen35_mi300x_agentx_mori_dep8_dep8_mtp + RUNNER_TYPE: mi300x + IMAGE: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910 + IS_MULTINODE: "true" + DISAGG: "true" + SPEC_DECODING: mtp + KV_OFFLOADING: none + PREFILL_NUM_WORKERS: "1" + PREFILL_TP: "8" + PREFILL_EP: "8" + PREFILL_DP_ATTN: "true" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "8" + DECODE_EP: "8" + DECODE_DP_ATTN: "true" + HF_HOME: /hf-cache + HF_HUB_CACHE: /hf-cache/hub + HUGGINGFACE_HUB_CACHE: /hf-cache/hub + PYTHONDONTWRITEBYTECODE: "1" + TOKENIZERS_PARALLELISM: "false" + TRANSFORMERS_VERBOSITY: error + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DRAIN_BACKEND: sglang + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + CLEAR_CACHE_BETWEEN_CONC: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi300x/agentic/disagg-1p1d-pure-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi300x/agentic/disagg-1p1d-pure-tp8-mtp.yaml new file mode 100644 index 0000000000..a2514a4783 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi300x/agentic/disagg-1p1d-pure-tp8-mtp.yaml @@ -0,0 +1,185 @@ +# AgentX Qwen3.5 FP8: native SGLang Router, MORI KV transfer, pure TP8/EP1 per role. +# Uses the srt-slurm AMD runtime and native Qwen3.5 MTP. +# Concurrency candidates are untuned; fast runs are not performance submissions. +# The shared launcher supplies the cluster profile and workspace/results mounts. +# srt-slurm and Pyxis own model caching and the container lifecycle. + +name: mi300x-qwen35-agentx-mori-disagg-1p1d-pure-tp8-mtp + +model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910 + precision: fp8 + +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + container: + image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910@sha256:985488daa99010f6c72f17286b05804165e26c683be932325487d10b4ea6fd49 + +slurm: + time_limit: "12:00:00" + +resources: + gpu_type: mi300x + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + prefill-policy: cache_aware + decode-policy: round_robin + cache-threshold: 0.3 + health-check-timeout-secs: 600 + health-check-interval-secs: 30 + +backend: + type: sglang + prefill_environment: &worker_environment + HF_HOME: /hf-cache + HF_HUB_CACHE: /hf-cache/hub + HUGGINGFACE_HUB_CACHE: /hf-cache/hub + PYTHONDONTWRITEBYTECODE: "1" + PYTHONUNBUFFERED: "1" + PYTHONNOUSERSITE: "1" + GLOO_SOCKET_IFNAME: ens50f1np1 + NCCL_SOCKET_IFNAME: ens50f1np1 + MORI_SOCKET_IFNAME: ens50f1np1 + MORI_RDMA_DEVICES: bnxt_re_bond0,bnxt_re_bond1,bnxt_re_bond2,bnxt_re_bond3,bnxt_re_bond4,bnxt_re_bond5,bnxt_re_bond6,bnxt_re_bond7 + IBDEVICES: bnxt_re_bond0,bnxt_re_bond1,bnxt_re_bond2,bnxt_re_bond3,bnxt_re_bond4,bnxt_re_bond5,bnxt_re_bond6,bnxt_re_bond7 + MORI_IB_GID_INDEX: "3" + MORI_RDMA_TC: "104" + MORI_RDMA_SL: "3" + MORI_IO_TC: "104" + MORI_IO_SL: "3" + NCCL_IB_TC: "104" + NCCL_IB_SL: "3" + NCCL_IB_GID_INDEX: "3" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000" + MORI_IO_QP_MAX_SEND_WR: "16384" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_SHMEM_MODE: ISOLATION + SGLANG_USE_AITER: "1" + SGLANG_USE_AITER_UNIFIED_ATTN: "1" + AITER_FLYDSL_FORCE: "1" + SGLANG_MAMBA_SSM_DTYPE: bfloat16 + SGLANG_MORI_QP_PER_TRANSFER: "4" + SGLANG_MORI_NUM_WORKERS: "4" + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_HEALTH_CHECK_TIMEOUT: "600" + SGLANG_TIMEOUT_KEEP_ALIVE: "1800" + decode_environment: + <<: *worker_environment + # AgentX throughput uses the existing Qwen3.5 native-MTP golden target. + # Retained for every AgentX run, including fast bring-up; evals disable them. + SGLANG_SIMULATE_ACC_LEN: "3.39" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + sglang_config: + prefill: &worker_config + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-a2a-backend: none + disaggregation-transfer-backend: mori + disaggregation-ib-device: bnxt_re_bond0,bnxt_re_bond1,bnxt_re_bond2,bnxt_re_bond3,bnxt_re_bond4,bnxt_re_bond5,bnxt_re_bond6,bnxt_re_bond7 + dtype: bfloat16 + # auto follows the explicit BF16 model activation dtype, not FP8 weights. + kv-cache-dtype: auto + attention-backend: aiter + # Image enables INT8 quick-reduce; do not combine with allreduce fusion. + enable-aiter-allreduce-fusion: false + page-size: 16 + mamba-ssm-dtype: bfloat16 + # Keep the native dense-MLP TP default; do not force fully-DP MLPs. + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + context-length: 262144 + disable-radix-cache: false + mem-fraction-static: 0.75 + max-running-requests: 64 + max-prefill-tokens: 32768 + chunked-prefill-size: 8192 + scheduler-recv-interval: 30 + tokenizer-worker-num: 6 + cuda-graph-max-bs-decode: 64 + watchdog-timeout: 3600 + enable-metrics: true + enable-cache-report: true + log-level: info + decode: + <<: *worker_config + chunked-prefill-size: 8192 + +srun_options: + container-writable: "" + container-remap-root: "" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +output: + record_launch_plan: true + +benchmark: + type: custom + command: | + set -euo pipefail + export PORT="${SRT_FRONTEND_PORT}" + export AIPERF_SERVER_URL="http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" + export RESULT_DIR="/results/${SLURM_JOB_ID}/agentic" + export AGENTIC_OUTPUT_DIR="/results/${SLURM_JOB_ID}" + mkdir -p "$RESULT_DIR" "$AGENTIC_OUTPUT_DIR" + exec bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + MODEL_PREFIX: qwen3.5 + FRAMEWORK: sglang-disagg + PRECISION: fp8 + CONC: "1" + CONC_LIST: "1 4" + DURATION: "3600" + RESULT_FILENAME: qwen35_mi300x_agentx_mori_pure_tp8_mtp + RUNNER_TYPE: mi300x + IMAGE: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910 + IS_MULTINODE: "true" + DISAGG: "true" + SPEC_DECODING: mtp + KV_OFFLOADING: none + PREFILL_NUM_WORKERS: "1" + PREFILL_TP: "8" + PREFILL_EP: "1" + PREFILL_DP_ATTN: "false" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "8" + DECODE_EP: "1" + DECODE_DP_ATTN: "false" + HF_HOME: /hf-cache + HF_HUB_CACHE: /hf-cache/hub + HUGGINGFACE_HUB_CACHE: /hf-cache/hub + PYTHONDONTWRITEBYTECODE: "1" + TOKENIZERS_PARALLELISM: "false" + TRANSFORMERS_VERBOSITY: error + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DRAIN_BACKEND: sglang + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + CLEAR_CACHE_BETWEEN_CONC: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi300x/agentic/disagg-1p1d-tep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi300x/agentic/disagg-1p1d-tep8-mtp.yaml new file mode 100644 index 0000000000..046c64d290 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi300x/agentic/disagg-1p1d-tep8-mtp.yaml @@ -0,0 +1,184 @@ +# AgentX Qwen3.5 FP8: native SGLang Router, MORI KV transfer, TP8/EP8 per role. +# Uses the srt-slurm AMD runtime and native Qwen3.5 MTP. +# Concurrency candidates are untuned; fast runs are not performance submissions. +# The shared launcher supplies the cluster profile and workspace/results mounts. +# srt-slurm and Pyxis own model caching and the container lifecycle. + +name: mi300x-qwen35-agentx-mori-disagg-1p1d-tep8-mtp + +model: + path: hf:Qwen/Qwen3.5-397B-A17B-FP8 + container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910 + precision: fp8 + +identity: + model: + repo: Qwen/Qwen3.5-397B-A17B-FP8 + container: + image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910@sha256:985488daa99010f6c72f17286b05804165e26c683be932325487d10b4ea6fd49 + +slurm: + time_limit: "12:00:00" + +resources: + gpu_type: mi300x + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: sglang + enable_multiple_frontends: false + args: + prefill-policy: cache_aware + decode-policy: round_robin + cache-threshold: 0.3 + health-check-timeout-secs: 600 + health-check-interval-secs: 30 + +backend: + type: sglang + prefill_environment: &worker_environment + HF_HOME: /hf-cache + HF_HUB_CACHE: /hf-cache/hub + HUGGINGFACE_HUB_CACHE: /hf-cache/hub + PYTHONDONTWRITEBYTECODE: "1" + PYTHONUNBUFFERED: "1" + PYTHONNOUSERSITE: "1" + GLOO_SOCKET_IFNAME: ens50f1np1 + NCCL_SOCKET_IFNAME: ens50f1np1 + MORI_SOCKET_IFNAME: ens50f1np1 + MORI_RDMA_DEVICES: bnxt_re_bond0,bnxt_re_bond1,bnxt_re_bond2,bnxt_re_bond3,bnxt_re_bond4,bnxt_re_bond5,bnxt_re_bond6,bnxt_re_bond7 + IBDEVICES: bnxt_re_bond0,bnxt_re_bond1,bnxt_re_bond2,bnxt_re_bond3,bnxt_re_bond4,bnxt_re_bond5,bnxt_re_bond6,bnxt_re_bond7 + MORI_IB_GID_INDEX: "3" + MORI_RDMA_TC: "104" + MORI_RDMA_SL: "3" + MORI_IO_TC: "104" + MORI_IO_SL: "3" + NCCL_IB_TC: "104" + NCCL_IB_SL: "3" + NCCL_IB_GID_INDEX: "3" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000" + MORI_IO_QP_MAX_SEND_WR: "16384" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_SHMEM_MODE: ISOLATION + SGLANG_USE_AITER: "1" + SGLANG_USE_AITER_UNIFIED_ATTN: "1" + AITER_FLYDSL_FORCE: "1" + SGLANG_MAMBA_SSM_DTYPE: bfloat16 + SGLANG_MORI_QP_PER_TRANSFER: "4" + SGLANG_MORI_NUM_WORKERS: "4" + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_HEALTH_CHECK_TIMEOUT: "600" + SGLANG_TIMEOUT_KEEP_ALIVE: "1800" + decode_environment: + <<: *worker_environment + # AgentX throughput uses the existing Qwen3.5 native-MTP golden target. + # Retained for every AgentX run, including fast bring-up; evals disable them. + SGLANG_SIMULATE_ACC_LEN: "3.39" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + sglang_config: + prefill: &worker_config + served-model-name: Qwen/Qwen3.5-397B-A17B-FP8 + trust-remote-code: true + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + disaggregation-transfer-backend: mori + disaggregation-ib-device: bnxt_re_bond0,bnxt_re_bond1,bnxt_re_bond2,bnxt_re_bond3,bnxt_re_bond4,bnxt_re_bond5,bnxt_re_bond6,bnxt_re_bond7 + dtype: bfloat16 + # auto follows the explicit BF16 model activation dtype, not FP8 weights. + kv-cache-dtype: auto + attention-backend: aiter + # Image enables INT8 quick-reduce; do not combine with allreduce fusion. + enable-aiter-allreduce-fusion: false + page-size: 16 + mamba-ssm-dtype: bfloat16 + moe-dense-tp-size: 1 + reasoning-parser: qwen3 + tool-call-parser: qwen3_coder + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + context-length: 262144 + disable-radix-cache: false + mem-fraction-static: 0.75 + max-running-requests: 64 + max-prefill-tokens: 32768 + chunked-prefill-size: 8192 + scheduler-recv-interval: 30 + tokenizer-worker-num: 6 + cuda-graph-max-bs-decode: 64 + watchdog-timeout: 3600 + enable-metrics: true + enable-cache-report: true + log-level: info + decode: + <<: *worker_config + chunked-prefill-size: 8192 + +srun_options: + container-writable: "" + container-remap-root: "" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +output: + record_launch_plan: true + +benchmark: + type: custom + command: | + set -euo pipefail + export PORT="${SRT_FRONTEND_PORT}" + export AIPERF_SERVER_URL="http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" + export RESULT_DIR="/results/${SLURM_JOB_ID}/agentic" + export AGENTIC_OUTPUT_DIR="/results/${SLURM_JOB_ID}" + mkdir -p "$RESULT_DIR" "$AGENTIC_OUTPUT_DIR" + exec bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + MODEL: Qwen/Qwen3.5-397B-A17B-FP8 + MODEL_PREFIX: qwen3.5 + FRAMEWORK: sglang-disagg + PRECISION: fp8 + CONC: "1" + CONC_LIST: "1 4 8 16" + DURATION: "3600" + RESULT_FILENAME: qwen35_mi300x_agentx_mori_tep8_mtp + RUNNER_TYPE: mi300x + IMAGE: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910 + IS_MULTINODE: "true" + DISAGG: "true" + SPEC_DECODING: mtp + KV_OFFLOADING: none + PREFILL_NUM_WORKERS: "1" + PREFILL_TP: "8" + PREFILL_EP: "8" + PREFILL_DP_ATTN: "false" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "8" + DECODE_EP: "8" + DECODE_DP_ATTN: "false" + HF_HOME: /hf-cache + HF_HUB_CACHE: /hf-cache/hub + HUGGINGFACE_HUB_CACHE: /hf-cache/hub + PYTHONDONTWRITEBYTECODE: "1" + TOKENIZERS_PARALLELISM: "false" + TRANSFORMERS_VERBOSITY: error + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DRAIN_BACKEND: sglang + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + CLEAR_CACHE_BETWEEN_CONC: "1" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b19a0390e1..890e58dcef 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1752,3 +1752,104 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 16], spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 48], spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [128, 256], spec-decoding: mtp } + +# MI300X AgentX: homogeneous attention layouts with stock MORI transfer. +qwen3.5-fp8-mi300x-sglang-disagg-dep8-agentic-mtp: + image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:mi300x-amd + precision: fp8 + framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + conc-list: [32, 64] + kv-offloading: none + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/mi300x/agentic/disagg-1p1d-dep8-mtp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_MTP_SIZE=3" + +# MI300X AgentX: homogeneous attention layouts with stock MORI transfer. +qwen3.5-fp8-mi300x-sglang-disagg-tep8-agentic-mtp: + image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:mi300x-amd + precision: fp8 + framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + conc-list: [1, 4, 8, 16] + kv-offloading: none + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/mi300x/agentic/disagg-1p1d-tep8-mtp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + additional-settings: + - "DECODE_MTP_SIZE=3" + +qwen3.5-fp8-mi300x-sglang-disagg-pure-tp8-agentic-mtp: + image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:mi300x-amd + precision: fp8 + framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + conc-list: [1, 4] + kv-offloading: none + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/mi300x/agentic/disagg-1p1d-pure-tp8-mtp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_MTP_SIZE=3" diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md index 09cb9666ba..6bb1ce17e5 100644 --- a/docs/eval-agentx-procedures.md +++ b/docs/eval-agentx-procedures.md @@ -260,6 +260,19 @@ The runner writes the command before replay and validates raw results after aggr ## 9. Debug long AgentX runs from live evidence +Native SGLang router recipes set `AIPERF_DRAIN_BACKEND=sglang`. Between points, +the client requires three idle polls across all advertised workers, including +running, waiting, and disaggregation transfer queues. Missing gauges or failed +L1/L2 cache flushes fail the native run; L3 clearing remains optional. Qwen3.5 +MI300X AgentX runs retain golden acceptance length 3.39, including fast bring-up. +Real-output diagnostics and evals are separate from golden-AL AgentX replay. +The shared srt-slurm adapter requires a separate `EVAL_ONLY=true` job when +evaluation is requested; it does not combine golden-AL throughput with accuracy +evaluation on the same workers. Cache flushing is explicit opt-in through +`CLEAR_CACHE_BETWEEN_CONC=1`, as set by the native MI300X recipes, and defaults +off for existing non-SGLang consumers. +Fast runs are not canonical frontier results. + GitHub Actions is the orchestration/final-status view. The cluster is the live diagnostic source. Obtain the SSH alias, runner user, and access-controlled paths from the InferenceX Clusters canvas. Never guess or publish private infrastructure coordinates. Resolve the exact matrix job: diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md index a9e9a077f7..01e23080de 100644 --- a/docs/eval-agentx-procedures_zh.md +++ b/docs/eval-agentx-procedures_zh.md @@ -258,6 +258,16 @@ Runner 会在 replay 前写入命令,并在聚合后校验原始结果([执 ## 9. 用实时证据调试长时间 AgentX 运行 +使用原生 SGLang router 的 recipe 应设置 `AIPERF_DRAIN_BACKEND=sglang`。 +客户端在不同并发点之间检查所有已公布的 worker,要求运行、等待及分离式传输队列 +连续三次轮询均为空。缺少必要指标或 L1/L2 缓存清理失败时,原生运行会报错; +L3 清理仍为可选操作。Qwen3.5 MI300X AgentX 运行(包括快速验证)均保留 +golden acceptance length 3.39。真实输出诊断和 eval 与 golden-AL AgentX replay +分开执行。共享 srt-slurm adapter 要求 eval 使用独立的 `EVAL_ONLY=true` 作业, +不在同一组 worker 上混合 golden-AL 吞吐测试和准确率评估。缓存清理必须通过 +`CLEAR_CACHE_BETWEEN_CONC=1` 显式启用,原生 MI300X recipe 已设置此项; +现有非 SGLang 使用方默认不执行缓存清理。快速运行不能作为 canonical Pareto frontier 结果。 + GitHub Actions 是 orchestration/最终状态视图;cluster 是实时诊断来源。从 InferenceX Clusters canvas 获取 SSH alias、runner user 和受访问控制的路径。绝不要猜测或公开私有基础设施坐标。 解析准确的矩阵作业: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f69af977ac..156c721a4e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7286,3 +7286,30 @@ description: - "Use thinking-on golden synthetic AL 3.51 for five-token DSpark throughput; disable adaptive verification and retain real verification for evals" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2974 + +- config-keys: + - qwen3.5-fp8-mi300x-sglang-disagg-dep8-agentic-mtp + no-evals: true + description: + - "Add focused MI300X Qwen3.5 AgentX MORI disaggregation: DEP8 prefill/decode at c32 and c64." + - "Use native srt-slurm coordination, cache-aware SGLang routing, the stock September 10 ROCm 7.2.0 image, and golden acceptance length 3.39; accuracy jobs disable acceptance simulation." + - "Keep attention layouts homogeneous because the pinned MORI implementation does not preserve Qwen convolution-state group ordering across attention TP8 to TP1." + - "Forward the recipe fingerprint into benchmark artifacts and retain existing workflow scheduling and full-sweep policy." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2984 + +- config-keys: + - qwen3.5-fp8-mi300x-sglang-disagg-pure-tp8-agentic-mtp + no-evals: true + description: + - "Add low-concurrency MI300X Qwen3.5 AgentX MORI disaggregation with pure TP8/EP1 prefill and decode at c1 and c4." + - "Keep the same stock ROCm 7.2.0 image, BF16 KV/Mamba state, native dense-MLP TP defaults, and golden acceptance length 3.39; validate accuracy separately without acceptance simulation." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2984 + +- config-keys: + - qwen3.5-fp8-mi300x-sglang-disagg-tep8-agentic-mtp + no-evals: true + description: + - "Add homogeneous MI300X Qwen3.5 AgentX 1P TEP8 / 1D TEP8 at session concurrencies 1, 4, 8 and 16 to the official full sweep." + - "Retain the proven screening recipe: stock ROCm 7.2.0, MORI transfer, TP8/EP8 on both roles, 8192-token prefill chunks, memory fraction 0.75, and golden acceptance length 3.39." + - "Use full default warmup and 3600-second profiling; skip evals as requested." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2984 diff --git a/runners/launch_mi300x-amd.sh b/runners/launch_mi300x-amd.sh index d2a656d237..2a3e9f4018 100644 --- a/runners/launch_mi300x-amd.sh +++ b/runners/launch_mi300x-amd.sh @@ -1,6 +1,14 @@ #!/usr/bin/env bash set -euo pipefail +if [[ -n "${CONFIG_FILE:-}" ]]; then + export SRT_SLURM_COMMIT="${SRT_SLURM_COMMIT:-c6dc2f05061e504c59b8e7baa30cfeacabc93645}" + export SRT_SLURM_CLUSTER_CONFIG="${SRT_SLURM_CLUSTER_CONFIG:-${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/cluster-configs/mi300x-amd.yaml}" + export SRT_SLURM_SHARED_BASE="${SRT_SLURM_SHARED_BASE:-$HOME/inferencex/srt-slurm}" + export AIPERF_MMAP_CACHE_HOST_PATH="${AIPERF_MMAP_CACHE_HOST_PATH:-$HOME/inferencex/aiperf-cache}" + exec bash "$(dirname "${BASH_SOURCE[0]}")/launch_srt_slurm.sh" +fi + export HF_HUB_CACHE_MOUNT="/raid/inferencex/models/hub" export AIPERF_MMAP_CACHE_MOUNT="/raid/inferencex/aiperf-mmap-cache" export AIPERF_DATASET_MMAP_CACHE_DIR="/aiperf_mmap_cache" diff --git a/runners/launch_srt_slurm.sh b/runners/launch_srt_slurm.sh new file mode 100644 index 0000000000..142d6bad39 --- /dev/null +++ b/runners/launch_srt_slurm.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Shared workflow adapter. Cluster entry points supply paths; srt-slurm owns +# allocation, container startup, workers, router, and benchmark execution. +: "${GITHUB_WORKSPACE:?GITHUB_WORKSPACE must be set by Actions}" +: "${RESULT_FILENAME:?RESULT_FILENAME must be set by the benchmark workflow}" +: "${CONFIG_FILE:?CONFIG_FILE must select an srt-slurm recipe}" +: "${IMAGE:?IMAGE must identify the serving container}" +: "${MODEL:?MODEL must identify the checkpoint}" +: "${SRT_SLURM_CLUSTER_CONFIG:?Cluster profile is required}" +: "${SRT_SLURM_SHARED_BASE:?Shared runtime directory is required}" +: "${AIPERF_MMAP_CACHE_HOST_PATH:?Shared AIPerf cache is required}" +SRT_SLURM_REPOSITORY="${SRT_SLURM_REPOSITORY:-https://github.com/SemiAnalysisAI/srt-slurm.git}" +SRT_SLURM_COMMIT="${SRT_SLURM_COMMIT:-81d46274f508e18ab14d1f123b75132005818dcf}" +RUN_KEY="${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-0}-${RUNNER_NAME:-runner}" +WORK_DIR=$(mktemp -d "${GITHUB_WORKSPACE}/.srt-slurm-${RUN_KEY}.XXXXXX") +SRT_REPO_DIR="${WORK_DIR}/srt-slurm" +SHARED_RESULTS="${SRT_SLURM_SHARED_BASE}/results" +CONFIG_PATH="${CONFIG_FILE%%:*}" +LOCAL_RECIPE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/${CONFIG_PATH#recipes/}" +ADAPTER="${GITHUB_WORKSPACE}/utils/srt_slurm.py" +mkdir -p "$SHARED_RESULTS" "$AIPERF_MMAP_CACHE_HOST_PATH" + +git clone "$SRT_SLURM_REPOSITORY" "$SRT_REPO_DIR" +git -C "$SRT_REPO_DIR" checkout --detach "$SRT_SLURM_COMMIT" +[[ "$(git -C "$SRT_REPO_DIR" rev-parse HEAD)" == "$SRT_SLURM_COMMIT" ]] +cd "$SRT_REPO_DIR" +make setup-compute ARCH="${SRT_SLURM_COMPUTE_ARCH:-x86_64}" +export PATH="$SRT_REPO_DIR/bin:$PATH" +uv venv --python 3.12 +uv pip install -e . +source .venv/bin/activate + +python "$ADAPTER" prepare \ + --recipe "$LOCAL_RECIPE" --profile "$SRT_SLURM_CLUSTER_CONFIG" \ + --work-dir "$WORK_DIR" --workspace "$GITHUB_WORKSPACE" \ + --results-root "$SHARED_RESULTS" --aiperf-cache "$AIPERF_MMAP_CACHE_HOST_PATH" \ + --image-cache "${SRT_SLURM_SHARED_BASE}/containers" +export SRTSLURM_CONFIG="${WORK_DIR}/srtslurm.yaml" +export SRTCTL_RUNTIME_SOURCE_DIR="$SRT_REPO_DIR" +PREPARED_RECIPE="${WORK_DIR}/recipe.yaml" +[[ "$CONFIG_FILE" != *:* ]] || PREPARED_RECIPE="${PREPARED_RECIPE}:${CONFIG_FILE#*:}" +SUBMISSION="${WORK_DIR}/submission.json" +JOB_ID="" +cleanup() { + local rc=$? collect_rc=0 + trap - EXIT INT TERM + if [[ -z "$JOB_ID" && -s "$SUBMISSION" ]]; then + JOB_ID=$(jq -er '.slurm_job_id' "$SUBMISSION") || JOB_ID="" + fi + if [[ "$JOB_ID" =~ ^[0-9]+$ ]]; then + # This ID comes only from this invocation's srtctl submission. + if [[ "$rc" -ne 0 ]]; then scancel "$JOB_ID" || true; fi + python "$ADAPTER" collect --submission "$SUBMISSION" \ + --workspace "$GITHUB_WORKSPACE" --results-root "$SHARED_RESULTS" || collect_rc=$? + [[ "$rc" -ne 0 ]] || rc=$collect_rc + fi + exit "$rc" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +# Keep the CI host's bytecode-cache path out of the container environment. +env -u PYTHONPYCACHEPREFIX srtctl apply -f "$PREPARED_RECIPE" \ + --tags "inferencex,github-actions,${RUN_KEY}" --json > "$SUBMISSION" +JOB_ID=$(jq -er '.slurm_job_id' "$SUBMISSION") +[[ "$JOB_ID" =~ ^[0-9]+$ ]] +OUTPUT_DIR=$(jq -er '.output_dir' "$SUBMISSION") +echo "SRT_SLURM_JOB_ID=$JOB_ID" +printf '%s\n' "$SRT_SLURM_COMMIT" > "$GITHUB_WORKSPACE/srt-slurm-producer-sha.txt" +srtctl wait "$JOB_ID" --log-file "${OUTPUT_DIR}/logs/sweep_${JOB_ID}.log" diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 778b357327..6b65a49bd7 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -9,6 +9,7 @@ from pydantic import BaseModel, ValidationError REPO_ROOT = Path(__file__).resolve().parents[1] +SRT_ADAPTER = REPO_ROOT / "utils" / "srt_slurm.py" SLURM_UTILS = REPO_ROOT / "runners" / "slurm_utils.sh" PATCH_SRT_EVAL = REPO_ROOT / "runners" / "patch_srt_eval_dispatch.py" PATCH_SRT_DP_RANKS = REPO_ROOT / "runners" / "patch_srt_vllm_dp_ranks.py" @@ -17,6 +18,167 @@ INJECT_ACCEPTANCE = REPO_ROOT / "runners" / "inject_synthetic_acceptance.py" +def test_srt_adapter_preserves_serving_recipe_and_uses_native_image_fallback(tmp_path: Path) -> None: + prepare = runpy.run_path(str(SRT_ADAPTER))["prepare_recipe"] + backend = {"type": "atom", "atom_config": {"prefill": {"kv_cache_dtype": "fp8", "max-num-seqs": 256}}} + recipe = { + "model": {"container": "model-image"}, + "backend": backend, + "benchmark": {"type": "custom", "command": "run-the-original-benchmark"}, + } + profile = {"default_mounts": {"/host/rdma": "/container/rdma"}} + paths = { + "workspace": tmp_path / "workspace", + "results_root": tmp_path / "results", + "aiperf_cache": tmp_path / "aiperf", + "image_cache": tmp_path / "images", + } + env = { + "IMAGE": "vendor/engine:pinned", + "CONC_LIST": "4 8", + "RECIPE_FINGERPRINT": "serving-recipe-digest", + "UNRELATED_SECRET": "do-not-forward", + } + prepared, cluster = prepare(recipe, profile, env, **paths) + + assert prepared["backend"] == backend + assert prepared["benchmark"]["command"] == "run-the-original-benchmark" + assert prepared["benchmark"]["env"] == { + "CONC_LIST": "4 8", "RECIPE_FINGERPRINT": "serving-recipe-digest" + } + assert "env" not in recipe["benchmark"] + assert profile == {"default_mounts": {"/host/rdma": "/container/rdma"}} + assert cluster["containers"] == {"model-image": "vendor/engine:pinned"} + assert cluster["default_mounts"][str(paths["workspace"])] == "/infmax-workspace" + paths["image_cache"].mkdir() + cached_image = paths["image_cache"] / "vendor_engine_pinned.sqsh" + cached_image.write_bytes(b"cached-image") + _, cached_cluster = prepare(recipe, profile, env, **paths) + assert cached_cluster["containers"] == {"model-image": str(cached_image)} + + +@pytest.mark.parametrize("aggregated", [False, True]) +def test_srt_adapter_preserves_explicit_sglang_settings_and_eval_contract(tmp_path: Path, aggregated: bool) -> None: + prepare = runpy.run_path(str(SRT_ADAPTER))["prepare_recipe"] + simulation = { + "SGLANG_SIMULATE_ACC_LEN": "3.39", + "SGLANG_SIMULATE_ACC_METHOD": "match-expected", + "SGLANG_SIMULATE_ACC_TOKEN_MODE": "real-draft-token", + } + recipe = { + "model": {"container": "image"}, + "backend": { + "type": "sglang", + "sglang_config": { + "prefill": { + "tp-size": 8, + "ep-size": 8, + "enable-dp-attention": True, + "context-length": 9472, + "max-running-requests": 256, + }, + "decode": { + "tp-size": 16, + "ep-dispatch-algorithm": "experimental", + "max-running-requests": 512, + }, + }, + "prefill_environment": dict(simulation), + "decode_environment": { + "SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "17", + "MORI_MAX_DISPATCH_TOKENS_DECODE": "1", + "SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD": "34", + **simulation, + }, + }, + "benchmark": {"type": "custom", "command": "original-throughput-command"}, + } + if aggregated: + backend = recipe["backend"] + worker = backend["sglang_config"]["prefill"] + worker["ep-dispatch-algorithm"] = "experimental" + backend["sglang_config"] = {"aggregated": worker} + backend["aggregated_environment"] = backend.pop("decode_environment") + backend.pop("prefill_environment") + env = { + "IMAGE": "vendor/engine:pinned", + "MODEL": "vendor/model", + "CONC_LIST": "16 64", + "PREFILL_DP_ATTN": "true", + "PREFILL_EP": "8", + "DECODE_TP": "16", + "DECODE_MTP_SIZE": "3", + } + paths = { + "workspace": tmp_path, + "results_root": tmp_path / "results", + "aiperf_cache": tmp_path / "cache", + "image_cache": tmp_path / "images", + } + throughput, _ = prepare(recipe, {}, env, **paths) + assert throughput["backend"] == recipe["backend"] + + with pytest.raises(ValueError, match="separate EVAL_ONLY=true job"): + prepare(recipe, {}, {**env, "RUN_EVAL": "true", "EVAL_ONLY": "false"}, **paths) + assert recipe["backend"] == throughput["backend"] + + evaluation, _ = prepare(recipe, {}, {**env, "EVAL_ONLY": "true", "EVAL_CONC": "64"}, **paths) + roles = ("aggregated",) if aggregated else ("prefill", "decode") + for role in roles: + role_env = evaluation["backend"][f"{role}_environment"] + assert not simulation.keys() & role_env.keys() + assert role_env == { + key: value for key, value in recipe["backend"][f"{role}_environment"].items() + if key not in simulation + } + assert "ep-dispatch-algorithm" not in evaluation["backend"]["sglang_config"][role] + assert evaluation["benchmark"]["env"]["MODEL_NAME"] == "vendor/model" + assert evaluation["benchmark"]["env"]["EVAL_MAX_MODEL_LEN"] == "9472" + assert "original-throughput-command" not in evaluation["benchmark"]["command"] + assert 'run_eval --port "${SRT_FRONTEND_PORT}"' in evaluation["benchmark"]["command"] + + # Current main stages AgentX eval-only output inside run_eval and removes + # the temporary directory. The adapter must not try to stage it again. + staging = evaluation["benchmark"]["command"].split('run_eval --port "${SRT_FRONTEND_PORT}"', 1)[1] + for eval_only, agentic, expected in (("true", "1", ""), ("false", "1", "staged"), ("true", "0", "staged")): + result = subprocess.run( + ["bash", "-c", 'append_lm_eval_summary() { printf staged; }; ' + staging], + env={**os.environ, "EVAL_ONLY": eval_only, "IS_AGENTIC": agentic}, + check=True, + capture_output=True, + text=True, + ) + assert result.stdout == expected + + +def test_srt_result_collection_is_job_scoped_and_preserves_artifact_names(tmp_path: Path) -> None: + collect = runpy.run_path(str(SRT_ADAPTER))["collect_results"] + workspace = tmp_path / "workspace" + workspace.mkdir() + results_root = tmp_path / "results" + for job_id in ("42", "99"): + fixed = results_root / job_id / "fixed-seq" + fixed.mkdir(parents=True) + (fixed / "benchmark-c8.json").write_text(json.dumps({"job": job_id})) + env = { + "RESULT_FILENAME": "benchmark", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "8", + "DECODE_NUM_WORKERS": "1", + "DECODE_TP": "8", + "DISAGG": "true", + } + submission = {"slurm_job_id": "42", "output_dir": str(tmp_path / "outputs" / "42")} + collect(submission, env, workspace=workspace, results_root=results_root) + artifact = workspace / "benchmark_srt-42_conc8_gpus_16_ctx_8_gen_8.json" + assert json.loads(artifact.read_text()) == {"job": "42"} + assert not list(workspace.glob("*srt-99*")) + + for flag in ("EVAL_ONLY", "RUN_EVAL"): + with pytest.raises(ValueError, match="No eval metadata"): + collect(submission, {**env, flag: "true"}, workspace=workspace, results_root=results_root) + + def run_bash(command: str, *args: Path | str) -> subprocess.CompletedProcess[str]: return subprocess.run( ["bash", "-c", command, "bash", *(str(arg) for arg in args)], diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index dad4914fa6..767f332415 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -1437,15 +1437,18 @@ def test_lm_patch_copy_resolves_outside_repo(tmp_path): """ -def _run_lm_eval_cmdline(*, eval_limit=None) -> str: +def _run_lm_eval_cmdline(*, eval_limit=None, eval_server_host=None) -> str: env = { **os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB), "KV_OFFLOADING": "none", } env.pop("EVAL_LIMIT", None) + env.pop("EVAL_SERVER_HOST", None) if eval_limit is not None: env["EVAL_LIMIT"] = str(eval_limit) + if eval_server_host is not None: + env["EVAL_SERVER_HOST"] = eval_server_host res = subprocess.run( ["bash", "-c", _EVAL_LIMIT_SCRIPT], env=env, @@ -1466,6 +1469,11 @@ def test_eval_limit_absent_when_unset(): assert "--limit" not in out, f"Expected no '--limit' in output:\n{out}" +def test_lm_eval_uses_routed_server_host_when_set(): + out = _run_lm_eval_cmdline(eval_server_host="10.0.0.42") + assert "base_url=http://10.0.0.42:9999/v1/chat/completions" in out + + def _summary_metadata(tmp_path: Path, **overrides: str) -> dict: work_dir = tmp_path / "work" results_dir = tmp_path / "results" @@ -2630,6 +2638,10 @@ def test_multinode_agentic_waits_only_for_eval_openai_endpoint( _wait_for_openai_chat_route() { echo "ready $*" >> "$EVENTS"; } build_replay_cmd() { echo build >> "$EVENTS"; } run_agentic_replay_and_write_outputs() { echo replay >> "$EVENTS"; } +curl() { + echo flush >> "$EVENTS" + printf 'Cache flushed 200' +} """, encoding="utf-8", ) @@ -2646,6 +2658,7 @@ def test_multinode_agentic_waits_only_for_eval_openai_endpoint( "RESULT_FILENAME": "result", "RESULT_DIR": str(tmp_path / "results"), "DURATION": "1", + "AIPERF_SERVER_METRICS_URLS": "http://worker.invalid:9000/metrics", } expected_without_readiness = ["resolve", "deps", "build", "replay"] @@ -2663,6 +2676,16 @@ def test_multinode_agentic_waits_only_for_eval_openai_endpoint( ) assert events_path.read_text().splitlines() == expected + events_path.unlink() + subprocess.run( + ["bash", str(MULTINODE_AGENTIC_SCRIPT)], + env={**base_env, "EVAL_ONLY": "false", "AIPERF_DRAIN_BACKEND": "sglang", "CLEAR_CACHE_BETWEEN_CONC": "1"}, + text=True, + capture_output=True, + check=True, + ) + assert events_path.read_text().splitlines() == ["resolve", "deps", "flush", "flush", "build", "replay"] + def test_env_can_force_bfcl_on_agentic_eval() -> None: output = _dispatch(is_agentic="1", eval_only="true", env_fw="bfcl") diff --git a/utils/srt_slurm.py b/utils/srt_slurm.py new file mode 100644 index 0000000000..bc8a76e6ea --- /dev/null +++ b/utils/srt_slurm.py @@ -0,0 +1,273 @@ +"""InferenceX recipe inputs and artifact contract for srt-slurm. + +This module does not allocate nodes, launch processes, poll Slurm, or repair +hosts. The cluster launcher supplies paths; srt-slurm owns the job lifecycle. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import os +import re +import shutil +import tarfile +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import yaml + +_FORWARDED_ENV = ( + "AIPERF_EXPERIMENTAL_FAST", + "CONC", + "CONC_LIST", + "DECODE_DP_ATTN", + "DECODE_EP", + "DECODE_NUM_WORKERS", + "DECODE_PCP_SIZE", + "DECODE_PP_SIZE", + "DECODE_TP", + "DURATION", + "EVAL_CONC", + "EVAL_FRAMEWORK", + "EVAL_LIMIT", + "EVAL_ONLY", + "EVAL_SUITE", + "FRAMEWORK", + "IS_AGENTIC", + "ISL", + "KV_OFFLOADING", + "MAX_MODEL_LEN", + "MODEL", + "MODEL_PREFIX", + "PREFILL_DP_ATTN", + "PREFILL_EP", + "PREFILL_NUM_WORKERS", + "PREFILL_PCP_SIZE", + "PREFILL_PP_SIZE", + "PREFILL_TP", + "PRECISION", + "RANDOM_RANGE_RATIO", + "RECIPE_FINGERPRINT", + "RESULT_FILENAME", + "RUN_EVAL", + "RUNNER_TYPE", + "OSL", + "SPEC_DECODING", + "SWEBENCH_GEN_MODE", + "TOTAL_CPU_DRAM_GB", +) + +_EVAL_COMMAND = r""" +set -euo pipefail +eval_root="/results/${SLURM_JOB_ID}/eval" +mkdir -p "${eval_root}" +cd "${eval_root}" +export SRTCTL_LM_EVAL_RESULT_DIR="${eval_root}" +source /infmax-workspace/benchmarks/benchmark_lib.sh +export EVAL_SERVER_HOST="${SRT_FRONTEND_HOST}" +if [[ -n "${EVAL_CONC:-}" ]]; then + export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" +else + export EVAL_CONCURRENT_REQUESTS="$(printf '%s\n' "${CONC_LIST:-${CONC:-1}}" | tr ' ' '\n' | sort -n | tail -1)" +fi +export CONC="${EVAL_CONCURRENT_REQUESTS}" +bridge_disagg_eval_metadata +run_eval --port "${SRT_FRONTEND_PORT}" +# AgentX eval-only runs stage their artifacts inside run_eval. +if [[ "${EVAL_ONLY:-false}" != "true" ]] || \ + [[ "${IS_AGENTIC:-0}" != "1" && "${SCENARIO_TYPE:-}" != "agentic-coding" ]]; then + append_lm_eval_summary +fi +""".strip() + + +def prepare_recipe( + recipe: dict[str, Any], + profile: dict[str, Any], + environment: Mapping[str, str], + *, + workspace: Path, + results_root: Path, + aiperf_cache: Path, + image_cache: Path, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Adapt CI metadata without changing the recipe's serving contract.""" + recipe = copy.deepcopy(recipe) + profile = copy.deepcopy(profile) + profile.setdefault("default_mounts", {}).update( + { + str(workspace): "/infmax-workspace", + str(results_root): "/results", + str(aiperf_cache): "/aiperf_mmap_cache", + } + ) + image = environment["IMAGE"] + cached_image = image_cache / (image.replace("/", "_").replace(":", "_") + ".sqsh") + # Reuse a provisioned image when available. Otherwise Pyxis imports the + # recipe's image during its normal container lifecycle, not a staging job. + profile.setdefault("containers", {})[recipe["model"]["container"]] = ( + str(cached_image) if cached_image.is_file() else image + ) + benchmark_env = recipe.setdefault("benchmark", {}).setdefault("env", {}) + for key in _FORWARDED_ENV: + value = environment.get(key) + if value: + benchmark_env[key] = value + + _configure_evaluation(recipe, environment) + return recipe, profile + + +def _configure_evaluation(recipe: dict[str, Any], environment: Mapping[str, str]) -> None: + benchmark_env = recipe["benchmark"]["env"] + eval_only = environment.get("EVAL_ONLY", "false").lower() == "true" + run_eval = environment.get("RUN_EVAL", "false").lower() == "true" + if run_eval and not eval_only: + raise ValueError("srt-slurm requires a separate EVAL_ONLY=true job for evaluation") + if eval_only: + benchmark_env["SRTCTL_LM_EVAL_RESULT_DIR"] = "/results/{job_id}/eval" + backend = recipe.get("backend", {}) + for role in ("prefill", "decode", "aggregated"): + role_env = backend.get(f"{role}_environment", {}) + for key in ( + "SGLANG_SIMULATE_ACC_LEN", + "SGLANG_SIMULATE_ACC_METHOD", + "SGLANG_SIMULATE_ACC_TOKEN_MODE", + ): + role_env.pop(key, None) + server_config = recipe.get("backend", {}).get("sglang_config", {}) + for mode in ("prefill", "decode", "aggregated"): + server_config.get(mode, {}).pop("ep-dispatch-algorithm", None) + + resources = recipe.get("resources", {}) + prefill = server_config.get("prefill", server_config.get("aggregated", {})) + decode = server_config.get("decode", prefill) + + def topology_value(config: dict[str, Any], *keys: str, default: int = 1) -> int: + for key in keys: + if key in config: + return int(config[key]) + return default + + topology_defaults = { + "IS_MULTINODE": "true", + "MODEL_NAME": environment["MODEL"], + "EVAL_MAX_MODEL_LEN": str(prefill.get("context-length", environment.get("MAX_MODEL_LEN", "16384"))), + "PREFILL_TP": str(topology_value(prefill, "tp-size", "tensor-parallel-size")), + "PREFILL_EP": str(topology_value(prefill, "ep-size", "expert-parallel-size")), + "PREFILL_NUM_WORKERS": str(resources.get("prefill_workers", resources.get("agg_workers", 1))), + "DECODE_TP": str(topology_value(decode, "tp-size", "tensor-parallel-size")), + "DECODE_EP": str(topology_value(decode, "ep-size", "expert-parallel-size")), + "DECODE_NUM_WORKERS": str(resources.get("decode_workers", resources.get("agg_workers", 1))), + "PREFILL_DP_ATTN": str(prefill.get("enable-dp-attention", False)).lower(), + "DECODE_DP_ATTN": str(decode.get("enable-dp-attention", False)).lower(), + } + for key, value in topology_defaults.items(): + benchmark_env.setdefault(key, value) + + recipe["benchmark"]["command"] = _EVAL_COMMAND + + +def collect_results( + submission: dict[str, Any], + environment: Mapping[str, str], + *, + workspace: Path, + results_root: Path, +) -> None: + """Collect only this allocation's artifacts into the workflow workspace.""" + job_id = str(submission["slurm_job_id"]) + if not job_id.isdecimal(): + raise ValueError("Submission must identify one numeric Slurm job") + log_dir = Path(submission["output_dir"]) / "logs" + result_dir = results_root / job_id + if log_dir.is_dir(): + with tarfile.open(workspace / "multinode_server_logs.tar.gz", "w:gz") as archive: + archive.add(log_dir, arcname=".") + lockfile = log_dir.parent / "recipe.lock.yaml" + if lockfile.is_file(): + archive.add(lockfile, arcname="recipe.lock.yaml") + if result_dir.is_dir(): + shutil.copytree(result_dir, workspace / "LOGS", dirs_exist_ok=True) + + filename = environment["RESULT_FILENAME"] + eval_only = environment.get("EVAL_ONLY", "false").lower() == "true" + if not eval_only and environment.get("IS_AGENTIC", "0") == "1": + for result in result_dir.glob(f"{filename}_conc*.json"): + shutil.copy2(result, workspace / result.name) + if not list(workspace.glob(f"{filename}_conc*.json")): + raise ValueError(f"No AgentX aggregate results found for {filename}") + elif not eval_only: + results = sorted((result_dir / "fixed-seq").glob("*.json")) + if not results: + raise ValueError(f"No fixed-sequence results found in {result_dir}") + prefill_gpus = int(environment["PREFILL_NUM_WORKERS"]) * int(environment["PREFILL_TP"]) + if environment.get("DISAGG", "false").lower() == "true": + decode_gpus = int(environment["DECODE_NUM_WORKERS"]) * int(environment["DECODE_TP"]) + suffix = f"gpus_{prefill_gpus + decode_gpus}_ctx_{prefill_gpus}_gen_{decode_gpus}" + else: + total = ( + prefill_gpus + * int(environment.get("PREFILL_PP_SIZE", "1")) + * int(environment.get("PREFILL_PCP_SIZE", "1")) + ) + suffix = f"gpus_{total}" + for result in results: + match = re.search(r"-c([0-9]+)\.json$", result.name) + if not match: + raise ValueError(f"Cannot parse concurrency from {result}") + destination = workspace / f"{filename}_srt-{job_id}_conc{match[1]}_{suffix}.json" + shutil.copy2(result, destination) + print(f"Collected {destination}") + + if eval_only or environment.get("RUN_EVAL", "false").lower() == "true": + eval_dir = result_dir / "eval" + if not (eval_dir / "meta_env.json").is_file(): + raise ValueError(f"No eval metadata found in {eval_dir}") + for artifact in eval_dir.glob("*"): + if artifact.is_file(): + shutil.copy2(artifact, workspace / artifact.name) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + prepare = commands.add_parser("prepare") + prepare.add_argument("--recipe", type=Path, required=True) + prepare.add_argument("--profile", type=Path, required=True) + prepare.add_argument("--work-dir", type=Path, required=True) + prepare.add_argument("--aiperf-cache", type=Path, required=True) + prepare.add_argument("--image-cache", type=Path, required=True) + collect = commands.add_parser("collect") + collect.add_argument("--submission", type=Path, required=True) + for command in (prepare, collect): + command.add_argument("--workspace", type=Path, required=True) + command.add_argument("--results-root", type=Path, required=True) + args = parser.parse_args() + if args.command == "prepare": + recipe, profile = prepare_recipe( + yaml.safe_load(args.recipe.read_text()), + yaml.safe_load(args.profile.read_text()), + os.environ, + workspace=args.workspace, + results_root=args.results_root, + aiperf_cache=args.aiperf_cache, + image_cache=args.image_cache, + ) + profile.setdefault("output_dir", str(args.work_dir / "outputs")) + (args.work_dir / "recipe.yaml").write_text(yaml.safe_dump(recipe, sort_keys=False)) + (args.work_dir / "srtslurm.yaml").write_text(yaml.safe_dump(profile, sort_keys=False)) + else: + collect_results( + json.loads(args.submission.read_text()), + os.environ, + workspace=args.workspace, + results_root=args.results_root, + ) + + +if __name__ == "__main__": + main()