From fdf83e7b9d5bbc986ebb38a76d2496c7a4f0c225 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 12 Oct 2026 02:24:41 +0000 Subject: [PATCH] cleanup(tools): delete finished tools/rdna2_028 campaign drivers The one-off campaign, probe, and matrix drivers under tools/rdna2_028 are finished and unreferenced by the live TunableOp row pipeline. Drop two doc how-to lines that only named deleted scripts. Co-authored-by: Cursor Agent Signed-off-by: Cursor Agent --- .../RESULTS-2026-09-30.md | 3 - docs/rdna2/w4a8-wiring.md | 4 +- tools/rdna2_028/analyze_captures.py | 118 ------- .../rdna2_028/audit_split/audit_split_swap.py | 35 --- tools/rdna2_028/audit_split/split_fixed.txt | 50 --- tools/rdna2_028/audit_split/split_old.txt | 14 - tools/rdna2_028/audit_w4a16.sh | 81 ----- tools/rdna2_028/build_rocm_c_incr.sh | 47 --- tools/rdna2_028/campaign_3a_capture.sh | 279 ----------------- tools/rdna2_028/campaign_3bcd_arm.sh | 100 ------ tools/rdna2_028/campaign_3bcd_curate.sh | 235 -------------- tools/rdna2_028/campaign_3d_freeze.sh | 106 ------- tools/rdna2_028/campaign_matrix.sh | 256 ---------------- tools/rdna2_028/exl3_analyze.py | 28 -- tools/rdna2_028/exl3_capture.sh | 247 --------------- tools/rdna2_028/exl3_freezer.py | 62 ---- tools/rdna2_028/exl3_mtp2_validate.sh | 137 --------- tools/rdna2_028/exl3_tunableop_campaign.sh | 192 ------------ tools/rdna2_028/exl3_validate.sh | 106 ------- tools/rdna2_028/flashnext_w4a8_arm.sh | 258 ---------------- tools/rdna2_028/flashnext_w4a8_extract.py | 92 ------ tools/rdna2_028/flashnext_w4a8_matrix.sh | 58 ---- tools/rdna2_028/fp32_accum_matrix.sh | 287 ------------------ tools/rdna2_028/fp_aroff_27b.sh | 148 --------- tools/rdna2_028/m27b_matrix.sh | 55 ---- tools/rdna2_028/parse_kernel_sass.py | 52 ---- tools/rdna2_028/pr30_b1_probe.sh | 127 -------- tools/rdna2_028/pr30_b2_rocprof.sh | 101 ------ tools/rdna2_028/pr30_b3_driver.sh | 112 ------- tools/rdna2_028/probe_force_split.py | 67 ---- tools/rdna2_028/probe_gptq_k.py | 69 ----- tools/rdna2_028/probe_inmodel_layer.py | 148 --------- tools/rdna2_028/probe_model_inspect2.py | 62 ---- tools/rdna2_028/probe_mtp_sampling.py | 62 ---- tools/rdna2_028/probe_prefill_split.py | 112 ------- tools/rdna2_028/probe_real_weights.py | 133 -------- tools/rdna2_028/probe_w4a8.py | 127 -------- tools/rdna2_028/probe_w4a8_moe.py | 172 ----------- tools/rdna2_028/probe_w4a8_op.py | 220 -------------- tools/rdna2_028/step0_parse.py | 102 ------- tools/rdna2_028/step0_profile.sh | 125 -------- tools/rdna2_028/step0_torchprof.sh | 93 ------ tools/rdna2_028/step0_tp_parse.py | 108 ------- tools/rdna2_028/tunableop_rows_e2e.sh | 34 --- tools/rdna2_028/w4a8_ab.sh | 96 ------ tools/rdna2_028/wt_ab.sh | 187 ------------ 46 files changed, 1 insertion(+), 5306 deletions(-) delete mode 100644 tools/rdna2_028/analyze_captures.py delete mode 100644 tools/rdna2_028/audit_split/audit_split_swap.py delete mode 100644 tools/rdna2_028/audit_split/split_fixed.txt delete mode 100644 tools/rdna2_028/audit_split/split_old.txt delete mode 100644 tools/rdna2_028/audit_w4a16.sh delete mode 100644 tools/rdna2_028/build_rocm_c_incr.sh delete mode 100755 tools/rdna2_028/campaign_3a_capture.sh delete mode 100755 tools/rdna2_028/campaign_3bcd_arm.sh delete mode 100755 tools/rdna2_028/campaign_3bcd_curate.sh delete mode 100755 tools/rdna2_028/campaign_3d_freeze.sh delete mode 100644 tools/rdna2_028/campaign_matrix.sh delete mode 100644 tools/rdna2_028/exl3_analyze.py delete mode 100755 tools/rdna2_028/exl3_capture.sh delete mode 100644 tools/rdna2_028/exl3_freezer.py delete mode 100644 tools/rdna2_028/exl3_mtp2_validate.sh delete mode 100755 tools/rdna2_028/exl3_tunableop_campaign.sh delete mode 100644 tools/rdna2_028/exl3_validate.sh delete mode 100644 tools/rdna2_028/flashnext_w4a8_arm.sh delete mode 100644 tools/rdna2_028/flashnext_w4a8_extract.py delete mode 100644 tools/rdna2_028/flashnext_w4a8_matrix.sh delete mode 100644 tools/rdna2_028/fp32_accum_matrix.sh delete mode 100644 tools/rdna2_028/fp_aroff_27b.sh delete mode 100755 tools/rdna2_028/m27b_matrix.sh delete mode 100644 tools/rdna2_028/parse_kernel_sass.py delete mode 100644 tools/rdna2_028/pr30_b1_probe.sh delete mode 100644 tools/rdna2_028/pr30_b2_rocprof.sh delete mode 100644 tools/rdna2_028/pr30_b3_driver.sh delete mode 100644 tools/rdna2_028/probe_force_split.py delete mode 100644 tools/rdna2_028/probe_gptq_k.py delete mode 100644 tools/rdna2_028/probe_inmodel_layer.py delete mode 100644 tools/rdna2_028/probe_model_inspect2.py delete mode 100644 tools/rdna2_028/probe_mtp_sampling.py delete mode 100644 tools/rdna2_028/probe_prefill_split.py delete mode 100644 tools/rdna2_028/probe_real_weights.py delete mode 100644 tools/rdna2_028/probe_w4a8.py delete mode 100644 tools/rdna2_028/probe_w4a8_moe.py delete mode 100644 tools/rdna2_028/probe_w4a8_op.py delete mode 100644 tools/rdna2_028/step0_parse.py delete mode 100644 tools/rdna2_028/step0_profile.sh delete mode 100644 tools/rdna2_028/step0_torchprof.sh delete mode 100644 tools/rdna2_028/step0_tp_parse.py delete mode 100644 tools/rdna2_028/tunableop_rows_e2e.sh delete mode 100644 tools/rdna2_028/w4a8_ab.sh delete mode 100644 tools/rdna2_028/wt_ab.sh diff --git a/docs/explore/w4a16-compile-dispatch/RESULTS-2026-09-30.md b/docs/explore/w4a16-compile-dispatch/RESULTS-2026-09-30.md index 0a2c8174b9d6..e8733bb76415 100644 --- a/docs/explore/w4a16-compile-dispatch/RESULTS-2026-09-30.md +++ b/docs/explore/w4a16-compile-dispatch/RESULTS-2026-09-30.md @@ -115,6 +115,3 @@ enough for the choice to matter). VLLM_USE_RDNA2_FA=1`. - Serve: `tools/rdna/serve_gfx1030_27b_dense.sh` with `CG_MODE=FULL_AND_PIECEWISE`, `MAXBAT=2048`, `SEQS=8`, `KV=8000000000`, `VLLM_DISABLE_COMPILE_CACHE=1`. -- Flash-Next: `tools/rdna2_028/flashnext_w4a8_arm.sh` with `W4A8=1`, - `CELLS="1 1024 512 221|8 1024 512 222"`, `GPUIDS=4,5,6,7`, - `VLLM_RDNA2_W4A8_DEBUG=1`. diff --git a/docs/rdna2/w4a8-wiring.md b/docs/rdna2/w4a8-wiring.md index 42c7a65a1f44..293fd85967e3 100644 --- a/docs/rdna2/w4a8-wiring.md +++ b/docs/rdna2/w4a8-wiring.md @@ -195,9 +195,7 @@ non-gfx1030). ### In-model A/B + matrix The serve launcher `tools/rdna/serve_gfx1030_27b_dense.sh` exports -`VLLM_RDNA2_W4A8_SDOT4` from the `W4A8` knob (default `1`). The matrix -driver `tools/rdna2_028/m27b_matrix.sh` runs 16k/1k and 1k/512 at c=1 and -c=8 for both `MTP=0` and `MTP=2`. Confirm the fast path fired by looking for +`VLLM_RDNA2_W4A8_SDOT4` from the `W4A8` knob (default `1`). Confirm the fast path fired by looking for the C++ `TORCH_WARN_ONCE` in the serve log: ```bash diff --git a/tools/rdna2_028/analyze_captures.py b/tools/rdna2_028/analyze_captures.py deleted file mode 100644 index 4667938b8edd..000000000000 --- a/tools/rdna2_028/analyze_captures.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Compare captured TunableOp shape traces against the frozen row sets. - -record_untuned (PYTORCH_TUNABLEOP_RECORD_UNTUNED=1) records EVERY GemmTunableOp -half-precision call the process makes, regardless of whether a row exists, so -the trace is a full observed-shape census. Cross-referencing it with the -shipped rows tells us: - - * covered -- key already in the curated row file (lookup will hit), - * dropped -- observed, present in an earlier (harvested) set but not in - the curated set: deliberately dropped because it reproduced - the heuristic or measured slower. Not a gap. - * novel -- observed in no row file we have ever shipped: a genuinely - uncaptured shape family and a tuning candidate. - - analyze_captures.py --captures --rows \ - [--backup ] [--out novel.txt] -""" - -from __future__ import annotations - -import argparse -import sys -from pathlib import Path - - -def keys_in_file(path: Path): - keys = set() - if not path or not path.exists(): - return keys - for line in path.read_text().splitlines(): - parts = line.split(",") - if len(parts) >= 2 and (parts[1].startswith("tn_") or parts[1].startswith("nn_")): - keys.add(parts[1]) - return keys - - -def load_capture(p: Path): - """shapes_*.txt are one bare key per line; untuned*.csv have a header col.""" - keys = set() - for line in p.read_text().splitlines(): - line = line.strip() - if not line or line.startswith("#"): - continue - tok = line.split(",")[-1] - if tok.startswith(("tn_", "nn_")): - keys.add(tok) - return keys - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--captures", required=True, type=Path, - help="dir holding shapes_*.txt (and/or untuned*.csv)") - ap.add_argument("--rows", required=True, type=Path, help="curated rows0.csv") - ap.add_argument("--backup", type=Path, default=None, - help="previously-shipped (harvested) rows0.csv") - ap.add_argument("--out", type=Path, default=None, help="write novel keys") - args = ap.parse_args() - - curated = keys_in_file(args.rows) - harvested = keys_in_file(args.backup) if args.backup else set() - print(f"curated rows : {len(curated)}") - if args.backup: - print(f"harvested rows: {len(harvested)}") - - per_arm = {} - def skip(f): - return any(part.startswith("_") for part in f.relative_to(args.captures).parts) - - for f in sorted(args.captures.rglob("shapes_*.txt")): - if skip(f): - continue - arm = f.parent.name if f.parent != args.captures else f.stem[len("shapes_"):] - per_arm.setdefault(arm, set()).update(load_capture(f)) - for f in sorted(args.captures.rglob("untuned[0-9].csv")): - if skip(f): - continue - arm = f.parent.name - per_arm.setdefault(arm, set()).update(load_capture(f)) - - if not per_arm: - print("no capture files found under", args.captures) - return 1 - - union = set().union(*per_arm.values()) - print(f"\n{'arm':28s} {'observed':>9s} {'covered':>8s} {'dropped':>8s} {'novel':>7s}") - for arm in sorted(per_arm): - obs = per_arm[arm] - cov = obs & curated - drp = (obs & harvested) - curated - nov = obs - curated - harvested - print(f"{arm:28s} {len(obs):9d} {len(cov):8d} {len(drp):8d} {len(nov):7d}") - - print("\npairwise set equality (by MTP):") - import itertools - for grp in ("m0", "m2"): - arms = sorted(a for a in per_arm if a.endswith(grp)) - for a, b in itertools.combinations(arms, 2): - same = per_arm[a] == per_arm[b] - only = len(per_arm[a] ^ per_arm[b]) - print(f" {a} == {b}: {same} (symdiff {only})") - - covered = union & curated - dropped = (union & harvested) - curated - novel = sorted(union - curated - harvested) - print(f"\n{'UNION':28s} {len(union):9d} {len(covered):8d} {len(dropped):8d} {len(novel):7d}") - print("\nnovel (not in curated and not in harvested):") - for k in novel: - print(" ", k) - if args.out: - args.out.write_text("\n".join(novel) + ("\n" if novel else "")) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/rdna2_028/audit_split/audit_split_swap.py b/tools/rdna2_028/audit_split/audit_split_swap.py deleted file mode 100644 index 1cc1566bf72a..000000000000 --- a/tools/rdna2_028/audit_split/audit_split_swap.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Swap the compute_split_k body in the scratch tree. - -Replaces the region between the end of the LDS-budget expression and the -VLLM_RDNA2_PREFILL_DEBUG print with a prepared block, so the same source tree -can be rebuilt with the legacy (pre-W4A8) split search, the W4A8-era -enumeration, or the repair (legacy-when-valid) implementation. -""" - -import argparse -import pathlib - -ANCHOR = ": (32 * 1024);\n\n" -END = " static const bool debug_split =" - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--file", required=True) - ap.add_argument("--block", required=True) - args = ap.parse_args() - - p = pathlib.Path(args.file) - txt = p.read_text() - block = pathlib.Path(args.block).read_text() - i = txt.index(ANCHOR) + len(ANCHOR) - j = txt.index(END) - p.write_text(txt[:i] + block + txt[j:]) - print(f"swapped {p} -> {pathlib.Path(args.block).name} ({j - i} -> {len(block)} bytes)") - - -if __name__ == "__main__": - main() diff --git a/tools/rdna2_028/audit_split/split_fixed.txt b/tools/rdna2_028/audit_split/split_fixed.txt deleted file mode 100644 index 34593179eaba..000000000000 --- a/tools/rdna2_028/audit_split/split_fixed.txt +++ /dev/null @@ -1,50 +0,0 @@ - // Repair, don't re-tune: keep the pre-W4A8 powers-of-two search whenever its - // choice is usable, so shapes that never hit the NaN bug keep their exact - // split and timing. The legacy choice is unusable only when its - // k_per_split = size_k/split is not a K_STEP multiple: the kernel advances K - // in K_STEP-wide chunks and never clamps the final chunk to k_per_split, so - // the tail walks past the split's LDS row and global K range (observed NaN / - // garbage at e.g. k=4352 -> split=16 -> k_per_split=272 at ConfigV1). - const int max_split_k = size_k / K_STEP; - int legacy = 1; - while (legacy < max_split_k && lds_bytes(legacy) > lds_budget) { - legacy *= 2; - } - while (legacy < 16 && max_split_k >= legacy * 2 && - (blocks_per_k_split * legacy < 2048 || - (size_k / legacy) > 2048)) { - const int candidate = legacy * 2; - if (lds_bytes(candidate) > lds_budget) break; - legacy = candidate; - } - int result; - if (size_k % legacy == 0 && (size_k / legacy) % K_STEP == 0) { - result = legacy; - } else { - // Fallback for the misaligned legacy choice only: enumerate every usable - // split <= 16 rather than only powers of two, so a k with few factors of - // two (e.g. 4160) still gets a valid split. `size_k % K_STEP == 0` (checked - // at entry) makes split=1 usable, so the list is never empty. - int splits[16]; - int count = 0; - for (int s = 1; s <= 16; ++s) { - if (size_k % s == 0 && (size_k / s) % K_STEP == 0) { - splits[count++] = s; - } - } - - int i = 0; - while (i + 1 < count && lds_bytes(splits[i]) > lds_budget) { - ++i; - } - while (i + 1 < count && - (blocks_per_k_split * splits[i] < 2048 || - (size_k / splits[i]) > 2048)) { - if (lds_bytes(splits[i + 1]) > lds_budget) { - break; - } - ++i; - } - result = splits[i]; - } - diff --git a/tools/rdna2_028/audit_split/split_old.txt b/tools/rdna2_028/audit_split/split_old.txt deleted file mode 100644 index 9deff954cdd2..000000000000 --- a/tools/rdna2_028/audit_split/split_old.txt +++ /dev/null @@ -1,14 +0,0 @@ - // Legacy (pre-W4A8) split search, restored verbatim for the scratch A/B. - const int max_split_k = size_k / K_STEP; - int result = 1; - while (result < max_split_k && lds_bytes(result) > lds_budget) { - result *= 2; - } - while (result < 16 && max_split_k >= result * 2 && - (blocks_per_k_split * result < 2048 || - (size_k / result) > 2048)) { - const int candidate = result * 2; - if (lds_bytes(candidate) > lds_budget) break; - result = candidate; - } - diff --git a/tools/rdna2_028/audit_w4a16.sh b/tools/rdna2_028/audit_w4a16.sh deleted file mode 100644 index 1eefe9dd30cd..000000000000 --- a/tools/rdna2_028/audit_w4a16.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# W4A16 regression-audit driver for the split-selection A/B. All artifacts land -# under $OUT// (persistent, no /tmp). Commands: -# -# probe run the op-level prefill timing/split probe (1 GPU) -# swap-old patch compute_split_k back to the pre-W4A8 search -# swap-fixed restore the repair (legacy-when-valid) implementation -# build incremental ninja rebuild of _rocm_C -# -# Typical sequence: -# audit_w4a16.sh probe new_build -# audit_w4a16.sh swap-old && audit_w4a16.sh build && audit_w4a16.sh probe old_build -# audit_w4a16.sh swap-fixed && audit_w4a16.sh build && audit_w4a16.sh probe fixed_build -set -uo pipefail - -V=${V:-/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0} -T=${T:-/home/chenco_adm/vllm-rdna-0.28.0} -OUT=${OUT:-/home/chenco_adm/w4a8_runs} -SRC=${SRC:-$T/csrc/rocm/q_gemm_rdna2_prefill.cu} -BLK=$T/tools/rdna2_028/audit_split - -ROCM_SDK_LIB="$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib" -ROCM_SDK="$V/lib/python3.12/site-packages/_rocm_sdk_core/lib" -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/rocm_sysdeps/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -export PYTHONPATH=$T -export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 -export VLLM_WORKER_MULTIPROC_METHOD=spawn GPU_MAX_HW_QUEUES=2 -export HIP_VISIBLE_DEVICES=${GPUIDS:-0} -export VLLM_ROCM_USE_AITER=0 FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE VLLM_RDNA_FORCE_FP16=1 -export TORCH_BLAS_PREFER_HIPBLASLT=0 VLLM_BATCH_INVARIANT=0 -# Rows live in the fork; never let TunableOp fall back to its default /tmp name. -# shellcheck source=tools/rdna2_028/tunableop_env.sh -source "$T/tools/rdna2_028/tunableop_env.sh" -configure_tunableop "$ROCM_SDK_LIB/librocblas.so.5" "$T/tunableop" -export VLLM_CACHE_ROOT=$T/cache/vllm -export TRITON_CACHE_DIR=$T/cache/triton -export TORCHINDUCTOR_CACHE_DIR=$T/cache/inductor - -cmd=${1:-} -tag=${2:-audit} -D=$OUT/$tag -mkdir -p "$D" - -case "$cmd" in - probe) - VLLM_RDNA2_PREFILL_DEBUG=1 "$V/bin/python" \ - "$T/tools/rdna2_028/probe_prefill_split.py" \ - --out "$D/prefill_timing.csv" --variants awq,gptq --iters 30 \ - >"$D/splits.log" 2>&1 - rc=$? - echo "[probe] rc=$rc csv=$D/prefill_timing.csv log=$D/splits.log" - exit $rc - ;; - swap-old) - "$V/bin/python" "$T/tools/rdna2_028/audit_split/audit_split_swap.py" \ - --file "$SRC" --block "$BLK/split_old.txt" - ;; - swap-fixed) - "$V/bin/python" "$T/tools/rdna2_028/audit_split/audit_split_swap.py" \ - --file "$SRC" --block "$BLK/split_fixed.txt" - ;; - build) - bash "$T/tools/rdna2_028/build_rocm_c_incr.sh" 2>&1 | tee "$D/build.log" - rc=${PIPESTATUS[0]} - if [ "$rc" = "0" ]; then - # ninja writes the linked module into the build dir; the editable install - # imports it from vllm/, so deploy it explicitly. - cp "$T/build/temp.linux-x86_64-cpython-312/_rocm_C.abi3.so" \ - "$T/vllm/_rocm_C.abi3.so" - echo "[build] deployed $(ls -la --time-style=+%H:%M "$T/vllm/_rocm_C.abi3.so")" - fi - exit $rc - ;; - *) - echo "usage: $0 {probe|swap-old|swap-fixed|build} [tag]" >&2 - exit 2 - ;; -esac diff --git a/tools/rdna2_028/build_rocm_c_incr.sh b/tools/rdna2_028/build_rocm_c_incr.sh deleted file mode 100644 index dd15c9fe38de..000000000000 --- a/tools/rdna2_028/build_rocm_c_incr.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# Incremental rebuild of _rocm_C via the cached cmake/ninja build dir. -# This is what the previous working build (w4a8_runs/build5.log) used: -# cmake has already configured build/temp.linux-x86_64-cpython-312, so a plain -# `ninja _rocm_C` recompiles only the changed TUs and relinks. Running -# `pip install -e .` instead re-configures from scratch and fails on the -# ROCm 7.14 non-standard SDK layout (/opt/rocm/core-7.14). -set -uo pipefail -T=${T:-/home/chenco_adm/vllm-rdna-0.28.0} -V=${V:-/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0} -BD=$T/build/temp.linux-x86_64-cpython-312 - -ROCM_SDK_LIB="$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib" -ROCM_SDK="$V/lib/python3.12/site-packages/_rocm_sdk_core/lib" -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/rocm_sysdeps/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -export PATH="$V/bin:/opt/rocm/core-7.14/bin:$PATH" -export CMAKE_HIP_COMPILER=/opt/rocm/core-7.14/lib/llvm/bin/clang++ -# The 7.14 SDK keeps the AMDGPU device bitcode under lib/llvm/amdgcn/bitcode -# (not /amdgcn/bitcode), so clang++ needs the override or every HIP -# compile dies with "cannot find ROCm device library". -export HIP_DEVICE_LIB_PATH=/opt/rocm/core-7.14/lib/llvm/amdgcn/bitcode -# NB: do NOT export HIP_CLANG_PATH here. cmake's link helper -# (hipcc_cmake_linker_helper) sets HIP_CLANG_PATH itself from the compiler dir -# baked into the link rule; an env value takes precedence in paths where the -# helper does not override it and makes hipcc exec "/clang++" (a -# "Not a directory" failure). - -# cmake's FindHIP hardcodes the standard /llvm/bin compiler dir when it -# generates the HIP link rule, but this 7.14 SDK keeps LLVM under -# /lib/llvm/bin, so the generated rule invokes a non-existent -# .../llvm/bin/clang++ and the link fails. Patch the generated rule (a build -# artifact, regenerated by cmake) to the real path. -RULES="$BD/CMakeFiles/rules.ninja" -if grep -q "/opt/rocm/core-7.14/llvm/bin" "$RULES" 2>/dev/null; then - sed -i 's#/opt/rocm/core-7.14/llvm/bin#/opt/rocm/core-7.14/lib/llvm/bin#g' "$RULES" - echo "[incr-build] patched HIP link rule -> lib/llvm/bin" -fi - -cd "$BD" -echo "[incr-build] start $(date +%H:%M:%S)" -ninja _rocm_C -rc=$? -echo "[incr-build] ninja_rc=$rc $(date +%H:%M:%S)" -if [ "$rc" = "0" ]; then - ls -la "$T/vllm/_rocm_C.abi3.so" -fi -exit $rc diff --git a/tools/rdna2_028/campaign_3a_capture.sh b/tools/rdna2_028/campaign_3a_capture.sh deleted file mode 100755 index 25063420ea9c..000000000000 --- a/tools/rdna2_028/campaign_3a_capture.sh +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# STEP 3 capture driver: 4-config x MTP=2 capture with record_untuned_enable. -# -# For each of {W4A16,W4A8} x {FA-RDNA2,Triton AMD FA}: -# * boot Flash-Next with the FROZEN rows active + record_untuned_enable=True -# pointed at a scratch filename. Any shape NOT in the 719-row set will be -# recorded (not tuned inline) for the offline tune pass. -# * run the four standard cells (c=1/c=8 x 1k/512 + 16k/1k). The cells data -# IS the STEP 4 validation, so this single pass serves both goals when no -# new shapes are captured. -# -# Outputs (per arm, under /home/chenco_adm/w4a8_runs//): -# serve.log, driver.log, coherence.txt, cells/c{IN}_{N}/{log, coherence.txt} -# untuned_captures.csv -- every GemmTunableOp key not in the frozen rows -# cells.csv, kfd.log, markers.txt -# -# No /tmp. Scratch rows live under /home/chenco_adm/w4a8_runs/_captures//. -set -uo pipefail - -V=/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0 -T=/home/chenco_adm/vllm-rdna-0.28.0 -MODEL=${MODEL:-/home/chenco_adm/hfcache/hub/models--wtdcode--Qwen3.8-Flash-Next-AWQ-W4A16/snapshots/0939125b929543a783ce700c90e36dd1a575c00c} -OUT=${OUT:-/home/chenco_adm/w4a8_runs} -ROWS=$T/tunableop/rocblas-f30bb442e9b5 -CAP_ROOT=$OUT/_captures -CELLS=${CELLS:-"1 1024 512 221|8 1024 512 222|1 16384 1024 111|8 16384 1024 112"} -# Arms to run in order. Default: all 4 at MTP=2 (the superset over MTP=0). -ARMS=${ARMS:-"w4a16-fa-m2 w4a16-triton-m2 w4a8-fa-m2 w4a8-triton-m2"} - -ROCM_SDK_LIB=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -ROCM_SDK=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/roCM_SYSDEPS/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$OUT/campaign_3a.log"; } - -# --- OUR processes only (0.28.0 venv) -------------------------------------- -our_pids() { - local p exe - for p in $(pgrep -f "entrypoints.cli.main serve|entrypoints.openai.api_server|VLLM::Worker|VLLM::EngineCore|PleOffloadWorker" 2>/dev/null); do - exe=$(readlink -f "/proc/$p/exe" 2>/dev/null || true) - case "$exe" in - */venv-7.14.0_0.28.0/bin/python*) echo "$p" ;; - esac - done -} - -hygiene() { - local n=0 left p - for p in $(our_pids); do kill -TERM "$p" 2>/dev/null && n=$((n + 1)); done - [ "$n" -gt 0 ] && sleep 8 - for p in $(our_pids); do kill -KILL "$p" 2>/dev/null; done - sleep 2 - left=$(our_pids | tr '\n' ' ') - log "hygiene: terminated=$n remaining_our=[${left% }]" - [ -n "${left// /}" ] && return 1 - return 0 -} - -sel_count() { timeout 8 sudo -n ipmitool sel list 2>/dev/null | grep -c 'PCI SERR'; } - -teardown() { - local api p - api=$(grep -oE "APIServer pid=[0-9]+" "$D/serve.log" 2>/dev/null | head -1 | cut -d= -f2) - [ -n "$api" ] && kill -TERM "$api" 2>/dev/null && sleep 10 - for p in $(our_pids); do kill -TERM "$p" 2>/dev/null; done - sleep 6 - for p in $(our_pids); do kill -KILL "$p" 2>/dev/null; done - sleep 2 -} - -run_arm() { - local arm=$1 - case "$arm" in - w4a16-fa-m2) W4A8=0 MTP=2 ATTN=fa PORT=18290 TAG=2026-09-30_w4a16-fa-m2-cap ;; - w4a16-triton-m2) W4A8=0 MTP=2 ATTN=triton PORT=18291 TAG=2026-09-30_w4a16-triton-m2-cap ;; - w4a8-fa-m2) W4A8=1 MTP=2 ATTN=fa PORT=18292 TAG=2026-09-30_w4a8-fa-m2-cap ;; - w4a8-triton-m2) W4A8=1 MTP=2 ATTN=triton PORT=18293 TAG=2026-09-30_w4a8-triton-m2-cap ;; - *) log "unknown arm: $arm"; return 1 ;; - esac - - local ARM=w4a8$([ "$W4A8" = 1 ] && echo on || echo off) - local CACHE=$T/cache/arm-cap-$arm - local TRITON_TEMPLATE=$T/cache/triton - local D=$OUT/$TAG - local CAP_DIR=$CAP_ROOT/$arm - mkdir -p "$D/cells" "$CAP_DIR" - - log "=== ARM=$arm W4A8=$W4A8 ATTN=$ATTN MTP=$MTP TAG=$TAG PORT=$PORT ===" - hygiene || { log "aborting: hygiene failed"; return 1; } - if ss -ltn 2>/dev/null | grep -q ":$PORT "; then - log "aborting: port $PORT bound"; return 1 - fi - rm -rf "$CACHE" - mkdir -p "$CACHE/inductor" "$CACHE/extensions" - if [ -d "$TRITON_TEMPLATE" ]; then - cp -a "$TRITON_TEMPLATE" "$CACHE/triton" - log "triton cache seeded from $TRITON_TEMPLATE ($(ls "$CACHE/triton" | wc -l) entries)" - else - mkdir -p "$CACHE/triton" - fi - rm -f "$CACHE/rdna_ar_wedged" - local SERR0; SERR0=$(sel_count) - log "arm cache=$CACHE baseline PCI-SERR=$SERR0 capture_dir=$CAP_DIR" - if [ "$SERR0" -gt "${CAMP_SERR_BASE:-100}" ]; then - log "STOP: PCI-SERR baseline $SERR0 > CAMP_SERR_BASE ${CAMP_SERR_BASE:-100}"; return 2 - fi - - local SCRATCH_DIR=$CAP_DIR/scratch - mkdir -p "$SCRATCH_DIR" - for r in 0 1 2 3; do - cp "$ROWS/tunableop_results${r}.csv" "$SCRATCH_DIR/tunableop_results${r}.csv" - done - - T0=$(date +%s) - setsid nohup env \ - VENV="$V" VLLM_TREE="$T" MTP="$MTP" ATTN="$ATTN" TP=4 PORT="$PORT" MODEL="$MODEL" \ - VLLM_CACHE_ROOT="$CACHE" TORCHINDUCTOR_CACHE_DIR="$CACHE/inductor" \ - TRITON_CACHE_DIR="$CACHE/triton" TORCH_EXTENSIONS_DIR="$CACHE/extensions" \ - VLLM_RDNA2_W4A8_SDOT4="$W4A8" \ - VLLM_RDNA_AR=1 VLLM_RDNA_AR_MAX_KB=64 VLLM_RDNA_AR_ONESHOT_KB=64 \ - VLLM_RDNA_AR_BLOCKS=0 VLLM_RDNA_AR_PACE=0 VLLM_FORCE_CUSTOM_ALL_REDUCE=0 \ - NCCL_P2P_LEVEL=pxb RCCL_P2P_NET_DISABLE=1 RCCL_P2P_BATCH_ENABLE=1 \ - NCCL_PROTO=Simple RCCL_MSCCL_ENABLE=0 PYTHONFAULTHANDLER=1 \ - HIP_VISIBLE_DEVICES=0,1,2,3 \ - PYTORCH_TUNABLEOP_ENABLED=1 PYTORCH_TUNABLEOP_TUNING=0 \ - PYTORCH_TUNABLEOP_HIPBLASLT_ENABLED=0 TORCH_BLAS_PREFER_HIPBLASLT=0 \ - PYTORCH_TUNABLEOP_FILENAME="$SCRATCH_DIR/tunableop_results%d.csv" \ - PYTORCH_TUNABLEOP_RECORD_UNTUNED=1 \ - bash "$T/tools/rdna/serve_gfx1030_flashnext_mtp.sh" >"$D/serve.log" 2>&1 /dev/null | grep -q flash-next && { ready=1; break; } - [ -f "$D/STOP" ] && break - if pgrep -f "[a]pi_server" >/dev/null; then dead=0; else dead=$((dead + 1)); fi - if [ "$dead" -ge 3 ]; then log "server process gone during boot (after $((i*10))s)"; break; fi - done - local COLD=$(( $(date +%s) - T0 )) - if [ "$ready" != "1" ]; then - log "NOT READY after ${COLD}s" - tail -40 "$D/serve.log" | tee -a "$D/driver.log" - teardown; hygiene - echo "NOT_READY" >"$D/status.txt" - return 1 - fi - log "READY t=${COLD}s" - echo "$COLD" >"$D/cold_compile_seconds.txt" - - # --- Warmup: throwaway 1k c=1 to clear cold-start pollution --- - for wl in "1024 64 999" "16384 64 998"; do - set -- $wl - log "warmup in=$1 out=$2" - ( cd "$D" && "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model "$MODEL" --served-model-name flash-next --dataset-name random \ - --random-input-len "$1" --random-output-len "$2" --num-prompts 1 --max-concurrency 1 \ - --ignore-eos --request-rate inf --seed "$3" --temperature 0 \ - --save-result --result-dir "$D/warmup_$1" ) >"$D/warmup_$1.log" 2>&1 - done - log "warmup done" - - # --- Coherence probe --- - log "coherence probe start" - "$V/bin/python" "$T/tools/rdna2_028/probe_w4a8.py" "http://127.0.0.1:$PORT/v1" flash-next 1 >"$D/coherence.txt" 2>&1 - log "coherence probe done rc=$?" - grep -E "^\[|OK /" "$D/coherence.txt" | tee -a "$D/driver.log" - - # --- Measured cells --- - IFS='|' read -ra _cells <<<"$CELLS" - echo "arm,dir,n,in_len,out_len,ok,dur_s,in_tok,out_tok,ttft_mean_ms,ttft_med_ms,ttft_p99_ms,tpot_mean_ms,itl_med_ms,itl_p99_ms,out_tok_s_agg,total_tok_s,decode_tok_s_per_req,prefill_tok_s" >"$D/cells.csv" - for cell in "${_cells[@]}"; do - set -- $cell - N=$1; IN=$2; OUTL=$3; SEED=$4 - local CD="$D/cells/c${IN}_${N}" - log "bench c=$N in=$IN out=$OUTL start" - ( cd "$D" && "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model "$MODEL" --served-model-name flash-next --dataset-name random \ - --random-input-len "$IN" --random-output-len "$OUTL" --num-prompts "$N" --max-concurrency "$N" \ - --ignore-eos --request-rate inf --seed "$SEED" --temperature 0 \ - --save-result --result-dir "$CD" ) >"$CD.log" 2>&1 - log "bench c=$N in=$IN done: $(grep -m1 'Output token throughput' "$CD.log" | tr -s ' ')" - # Per-cell coherence (France -> Paris, 2+2 -> 4) - "$V/bin/python" - "$PORT" "$CD" <<'PY' >"$CD/coherence.txt" 2>&1 -import json, sys, urllib.request -port, out = sys.argv[1], sys.argv[2] -def ask(prompt, mt=16): - body = json.dumps({"model": "flash-next", "prompt": prompt, "max_tokens": mt, - "temperature": 0.0, "ignore_eos": True}).encode() - req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions", data=body, - headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(req, timeout=600) as r: - return json.load(r)["choices"][0]["text"] or "" -for name, prompt, expect in [("france", "The capital of France is", "paris"), - ("math", "2 + 2 =", "4")]: - try: - t = ask(prompt) - except Exception as e: - t = f"" - ok = expect in t.lower() - print(f"[{name}] coherent={ok} | {t[:120]!r}") -PY - done - - # --- Markers --- - { - echo "=== 'W4A8 sdot4 path active' (expected iff W4A8=1 AND a dense quantized linear ran) ===" - grep -m4 "W4A8 sdot4 path active" "$D/serve.log" || echo "(absent)" - echo "=== 'W4A8-DEBUG' fast-path shapes ===" - grep -h "W4A8-DEBUG" "$D/serve.log" | sort -u | head -30 || echo "(none)" - echo "=== rdna_ar lines (expected: one-shot active on every rank) ===" - grep -h "rdna_ar:" "$D/serve.log" | sort -u || echo "(none)" - echo "=== all-reduce backend selection ===" - grep -m2 "all-reduce backends" "$D/serve.log" || echo "(none)" - echo "=== attention backend ===" - grep -m2 "attention backend" "$D/serve.log" | head -4 || echo "(none)" - echo "=== TunableOp record_untuned_enable messages ===" - grep -h "record_untuned\|TunableOp" "$D/serve.log" | sort -u | head -10 || echo "(none)" - } >"$D/markers.txt" 2>&1 - tee -a "$D/driver.log" <"$D/markers.txt" - - if [ -d "$SCRATCH_DIR" ]; then - for r in 0 1 2 3; do - f=$SCRATCH_DIR/tunableop_results${r}.csv - if [ -f "$f" ]; then - local cap_lines; cap_lines=$(wc -l <"$f") - local gemm_keys; gemm_keys=$(grep -c "^GemmTunableOp" "$f" 2>/dev/null || echo 0) - log "capture rank $r: $f ($cap_lines lines, $gemm_keys gemm keys)" - fi - done - else - log "capture scratch dir: $SCRATCH_DIR ABSENT" - fi - - # --- Teardown --- - teardown - sleep 3 - local left; left=$(our_pids | tr '\n' ' ') - log "teardown done remaining_our=[${left% }]" - echo "OK" >"$D/status.txt" -} - -# --- Driver loop ----------------------------------------------------------- -SERR0=$(sel_count) -log "=== campaign 3a start: ARMS=$ARMS PCI-SERR=$SERR0 ===" -log "preflight uptime=$(uptime -p) host=$(hostname)" -hygiene || { log "hygiene failed at start"; exit 1; } - -local_rc=0 -for arm in $ARMS; do - log "---------------------------------------------------------------" - if ! run_arm "$arm"; then - log "ARM FAIL: $arm (continuing)" - local_rc=1 - hygiene || { log "hygiene failed after $arm"; break; } - fi -done - -SERR1=$(sel_count) -log "=== campaign 3a done: rc=$local_rc PCI-SERR=$SERR1 (base $SERR0) ===" -[ "$SERR1" -gt "$SERR0" ] && { echo "STOP NEW PCI SERR" >"$OUT/STOP"; log "STOP: new PCI SERR"; exit 2; } - -# Aggregate capture summary -echo "=== capture summary ===" -for arm in $ARMS; do - f=$CAP_ROOT/$arm/untuned_captures.csv - if [ -f "$f" ]; then - n=$(grep -c "^GemmTunableOp" "$f" 2>/dev/null || echo 0) - echo "$arm: $n captured GemmTunableOp keys" - fi -done - -exit $local_rc diff --git a/tools/rdna2_028/campaign_3bcd_arm.sh b/tools/rdna2_028/campaign_3bcd_arm.sh deleted file mode 100755 index 274ce3874eda..000000000000 --- a/tools/rdna2_028/campaign_3bcd_arm.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# Tune + measure + curate the new shapes captured by the parallel -# `tools/rdna2_028/campaign_matrix.sh` capture phase. Uses a single GPU -# (default cuda:0) so the parallel campaign's TP=4 boot on GPUs 0-3 stays -# healthy; tune is still serial on the device, so 1 GPU is enough. -set -uo pipefail - -V=/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0 -T=/home/chenco_adm/vllm-rdna-0.28.0 -WORK=/home/chenco_adm/w4a8_runs/tunableop-campaign -CAP_DIR=${CAP_DIR:-/home/chenco_adm/w4a8_runs/_captures/w4a16-fa-m0} -ARM_NAME=${ARM_NAME:-w4a16-fa-m0} -ROWS=$T/tunableop/rocblas-f30bb442e9b5 -DEVICE=${DEVICE:-0} - -mkdir -p "$WORK" -ROCM_SDK_LIB=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -ROCM_SDK=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/rocm_sysdeps/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -export PYTORCH_TUNABLEOP_HIPBLASLT_ENABLED=0 TORCH_BLAS_PREFER_HIPBLASLT=0 -export HIP_VISIBLE_DEVICES=$DEVICE - -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$WORK/driver_${ARM_NAME}.log"; } - -log "=== tune+curate ARM=$ARM_NAME HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES (device=$DEVICE in scope) work=$WORK ===" - -awk -F, '$1 ~ /^GemmTunableOp_Half/' "$CAP_DIR/shapes_${ARM_NAME}.txt" 2>/dev/null \ - | awk -F, '{print $2}' | sort -u >"$WORK/cap_keys_${ARM_NAME}.txt" -awk -F, '$1 ~ /^GemmTunableOp/ {print $2}' "$ROWS/tunableop_results0.csv" \ - | sort -u >"$WORK/src_keys_${ARM_NAME}.txt" -comm -23 "$WORK/cap_keys_${ARM_NAME}.txt" "$WORK/src_keys_${ARM_NAME}.txt" \ - >"$WORK/new_keys_${ARM_NAME}.txt" -N_NEW=$(wc -l <"$WORK/new_keys_${ARM_NAME}.txt") -log "source rows: $(wc -l <$WORK/src_keys_${ARM_NAME}.txt); cap keys: $(wc -l <$WORK/cap_keys_${ARM_NAME}.txt); new: $N_NEW" - -if [ "$N_NEW" -eq 0 ]; then - log "no new shapes -> skip tune/curate" - exit 0 -fi - -TUNE_SRC=$WORK/tune_source_${ARM_NAME}.csv -echo "Validator,PT_VERSION,2.12.0" >"$TUNE_SRC" -echo "Validator,HIP_VERSION,714" >>"$TUNE_SRC" -while read -r key; do - [ -z "$key" ] && continue - echo "GemmTunableOp_Half_TN,$key,Default,0.001" >>"$TUNE_SRC" -done <"$WORK/new_keys_${ARM_NAME}.txt" -log "tune source: $TUNE_SRC ($N_NEW keys)" - -TUNE_SCRATCH=$WORK/scratch_${ARM_NAME} -mkdir -p "$TUNE_SCRATCH" -log "tune device=$DEVICE -> $TUNE_SCRATCH/tunableop_results0.csv" -"$V/bin/python" "$T/tools/rdna2_028/tune_prod_shapes.py" \ - --shape-source "$TUNE_SRC" --mode tune --device "$DEVICE" \ - --scratch "$TUNE_SCRATCH" --rank 0 --no-small-m \ - --iterations 10 --max-ms 25 \ - >"$WORK/tune_${ARM_NAME}.out" 2>"$WORK/tune_${ARM_NAME}.err" \ - || { log "TUNE FAIL"; tail -20 "$WORK/tune_${ARM_NAME}.err"; exit 1; } -log "tune done rows=$(grep -c '^GemmTunableOp' $TUNE_SCRATCH/tunableop_results0.csv 2>/dev/null || echo 0)" - -MEAS_SRC=$WORK/measure_source_${ARM_NAME}.csv -cp "$ROWS/tunableop_results0.csv" "$MEAS_SRC" -while read -r key; do - [ -z "$key" ] && continue - echo "GemmTunableOp_Half_TN,$key,Default,0.001" >>"$MEAS_SRC" -done <"$WORK/new_keys_${ARM_NAME}.txt" -log "measure source: $MEAS_SRC ($(grep -c "^GemmTunableOp" $MEAS_SRC) total)" - -# 3. Measure: heuristic / current / new -MEAS=$WORK/measure_${ARM_NAME} -mkdir -p "$MEAS" -for cond in heuristic current new; do - case $cond in - heuristic) src=(--heuristic) ;; - current) src=(--rows "$ROWS/tunableop_results0.csv") ;; - new) src=(--rows "$TUNE_SCRATCH/tunableop_results0.csv") ;; - esac - log "measure $cond" - "$V/bin/python" "$T/tools/rdna2_028/tune_prod_shapes.py" \ - --shape-source "$MEAS_SRC" --mode measure --device "$DEVICE" "${src[@]}" \ - --warmup 5 --reps 15 --out "$MEAS/$cond.json" \ - >"$WORK/measure_${ARM_NAME}_$cond.out" 2>"$WORK/measure_${ARM_NAME}_$cond.err" \ - || { log "MEASURE FAIL $cond"; tail -20 "$WORK/measure_${ARM_NAME}_$cond.err"; exit 1; } -done - -# 4. Curate -log "curate" -"$V/bin/python" "$T/tools/rdna2_028/curate_tunableop_rows.py" \ - --current-rows "$ROWS" --scratch-rows "$TUNE_SCRATCH" \ - --heur "$MEAS/heuristic.json" --cur "$MEAS/current.json" --new "$MEAS/new.json" \ - --adopt-margin 0.03 --drop-margin 0.03 --ranks 0,1,2,3 \ - --out "$WORK/curated_${ARM_NAME}" \ - >"$WORK/curate_${ARM_NAME}.log" 2>&1 \ - || { log "CURATE FAIL"; cat "$WORK/curate_${ARM_NAME}.log"; exit 1; } -grep -E "^decisions" "$WORK/curate_${ARM_NAME}.log" | tee -a "$WORK/driver_${ARM_NAME}.log" - -log "tune+curate ARM=$ARM_NAME done" diff --git a/tools/rdna2_028/campaign_3bcd_curate.sh b/tools/rdna2_028/campaign_3bcd_curate.sh deleted file mode 100755 index 811bfb1ce071..000000000000 --- a/tools/rdna2_028/campaign_3bcd_curate.sh +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# STEP 3 post-capture orchestration: -# 1. Diff per-arm scratch rows vs the frozen source rows to identify NEW shapes. -# 2. If deltas exist: tune them (PHASE=tune), then curate (PHASE=measure + curate). -# 3. If no deltas: skip the tune phase; the existing rows already cover the -# captured configs. -# 4. Freeze the merged/curated rows into tunableop/rocblas-f30bb442e9b5/ -# (4 ranks) and update provenance.json. -# -# Caller is responsible for committing + pushing after this script writes -# the rows. No /tmp; everything under $WORK. -set -uo pipefail - -V=${VENV:-/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0} -T=${VLLM_TREE:-/home/chenco_adm/vllm-rdna-0.28.0} -WORK=${WORK:-/home/chenco_adm/w4a8_runs/tunableop-campaign} -ROWS=$T/tunableop/rocblas-f30bb442e9b5 -CAP_ROOT=${CAP_ROOT:-/home/chenco_adm/w4a8_runs/_captures} -ARMS=${ARMS:-"w4a16-fa-m2 w4a16-triton-m2 w4a8-fa-m2 w4a8-triton-m2"} -PHASE=${PHASE:-all} # diff|tune|curate|freeze|all - -mkdir -p "$WORK" -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$WORK/driver.log"; } - -ROCM_SDK_LIB=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -ROCM_SDK=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/rocm_sysdeps/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -export PYTORCH_TUNABLEOP_HIPBLASLT_ENABLED=0 TORCH_BLAS_PREFER_HIPBLASLT=0 - -# --- 1. DIFF: collect all captured keys per arm and union them ------------ -diff_phase() { - log "=== PHASE=diff ===" - local union_keys=$WORK/captured_keys.txt - : >"$union_keys" - local per_arm=$WORK/per_arm_keys.txt - : >"$per_arm" - local total_captured=0 - - for arm in $ARMS; do - local f=$CAP_ROOT/$arm/scratch/tunableop_results0.csv - if [ ! -f "$f" ]; then - log "$arm: no scratch rows, skipping" - continue - fi - local cur=$(grep -c "^GemmTunableOp" "$f" || echo 0) - log "$arm: scratch rows=$cur" - awk -F, '$1 ~ /^GemmTunableOp/ {print $2}' "$f" | sort -u >>"$union_keys" - echo "$arm $(grep -c "^GemmTunableOp" "$f")" >>"$per_arm" - done - - sort -u "$union_keys" -o "$union_keys" - local n_captured=$(wc -l <"$union_keys") - - awk -F, '$1 ~ /^GemmTunableOp/ {print $2}' "$ROWS/tunableop_results0.csv" | \ - sort -u >"$WORK/source_keys.txt" - local n_source=$(wc -l <"$WORK/source_keys.txt") - - comm -23 "$union_keys" "$WORK/source_keys.txt" >"$WORK/new_keys.txt" - local n_new=$(wc -l <"$WORK/new_keys.txt") - log "source rows: $n_source; captured (union over $ARMS): $n_captured; new: $n_new" - - if [ "$n_new" -gt 0 ]; then - log "NEW shapes detected:" - head -20 "$WORK/new_keys.txt" | tee -a "$WORK/driver.log" - [ "$n_new" -gt 20 ] && log "... ($((n_new - 20)) more)" - fi -} - -# --- 2. TUNE: offline tune the new shapes (only if diff found deltas) ---- -tune_phase() { - local n_new=$(wc -l <"$WORK/new_keys.txt" 2>/dev/null || echo 0) - if [ "$n_new" -eq 0 ]; then - log "=== PHASE=tune SKIPPED (no new shapes) ===" - return 0 - fi - log "=== PHASE=tune ($n_new new shapes) ===" - - # Build a shape-source file containing source rows + new keys - local SHAPE_SRC=$WORK/shape_source.csv - cat "$ROWS/tunableop_results0.csv" >"$SHAPE_SRC" - while read -r key; do - [ -z "$key" ] && continue - echo "GemmTunableOp_Half_TN,$key,Default,0.001" >>"$SHAPE_SRC" - done <"$WORK/new_keys.txt" - - local TUNE_SCRATCH=$WORK/scratch - mkdir -p "$TUNE_SCRATCH" - local pids=() - for gpu in 0 1 2 3; do - log "tune gpu=$gpu -> $TUNE_SCRATCH/tunableop_results$gpu.csv (parallel)" - "$V/bin/python" "$T/tools/rdna2_028/tune_prod_shapes.py" \ - --shape-source "$SHAPE_SRC" --mode tune --device "$gpu" \ - --scratch "$TUNE_SCRATCH" --rank "$gpu" --no-small-m \ - --iterations 10 --max-ms 25 \ - >"$WORK/tune_gpu$gpu.json" 2>"$WORK/tune_gpu$gpu.err" & - pids+=($!) - done - local rc=0 - for i in 0 1 2 3; do - wait "${pids[$i]}" || { log "TUNE FAIL gpu=$i"; tail -20 "$WORK/tune_gpu$i.err"; rc=1; } - done - [ "$rc" -ne 0 ] && exit 1 - for gpu in 0 1 2 3; do - log "tune gpu=$gpu done rows=$(grep -c "^GemmTunableOp" "$TUNE_SCRATCH/tunableop_results$gpu.csv" 2>/dev/null || echo 0)" - done -} - -# --- 3. CURATE: heuristic / current / new A/B per shape, then decide ----- -curate_phase() { - local n_new=$(wc -l <"$WORK/new_keys.txt" 2>/dev/null || echo 0) - if [ "$n_new" -eq 0 ]; then - log "=== PHASE=curate SKIPPED (no new shapes; existing rows unchanged) ===" - echo "no_new_shapes" >"$WORK/curate_status.txt" - return 0 - fi - - log "=== PHASE=curate ===" - - # Union shape source: existing rows + new keys (so the A/B measures each one) - local SHAPE_SRC=$WORK/shape_source.csv - cat "$ROWS/tunableop_results0.csv" >"$SHAPE_SRC" - while read -r key; do - [ -z "$key" ] && continue - echo "GemmTunableOp_Half_TN,$key,Default,0.001" >>"$SHAPE_SRC" - done <"$WORK/new_keys.txt" - - local MEAS=$WORK/measure - mkdir -p "$MEAS" - for cond in heuristic current new; do - local src=() - case $cond in - heuristic) src=(--heuristic) ;; - current) src=(--rows "$ROWS/tunableop_results0.csv") ;; - new) src=(--rows "$WORK/scratch/tunableop_results0.csv") ;; - esac - log "measure $cond" - "$V/bin/python" "$T/tools/rdna2_028/tune_prod_shapes.py" \ - --shape-source "$SHAPE_SRC" --mode measure --device 0 "${src[@]}" \ - --warmup 5 --reps 15 --out "$MEAS/$cond.json" \ - >"$WORK/measure_$cond.out" 2>"$WORK/measure_$cond.err" \ - || { log "MEASURE FAIL $cond"; tail -20 "$WORK/measure_$cond.err"; exit 1; } - done - - "$V/bin/python" "$T/tools/rdna2_028/curate_tunableop_rows.py" \ - --current-rows "$ROWS" --scratch-rows "$WORK/scratch" \ - --heur "$MEAS/heuristic.json" --cur "$MEAS/current.json" --new "$MEAS/new.json" \ - --adopt-margin 0.03 --drop-margin 0.03 --ranks 0,1,2,3 \ - --out "$WORK/curated" \ - >"$WORK/curate.log" 2>&1 \ - || { log "CURATE FAIL"; cat "$WORK/curate.log"; exit 1; } - grep -E "^decisions" "$WORK/curate.log" | tee -a "$WORK/driver.log" - echo "curated" >"$WORK/curate_status.txt" -} - -# --- 4. FREEZE: copy curated rows into tunableop/, update provenance ------ -freeze_phase() { - log "=== PHASE=freeze ===" - local status=$(cat "$WORK/curate_status.txt" 2>/dev/null || echo "no_new_shapes") - - if [ "$status" = "no_new_shapes" ]; then - log "freeze SKIPPED: no new shapes; existing rows unchanged" - echo "no_freeze_needed" >"$WORK/freeze_status.txt" - return 0 - fi - - for r in 0 1 2 3; do - cp "$WORK/curated/tunableop_results${r}.csv" "$ROWS/tunableop_results${r}.csv" - local n=$(($(wc -l <"$ROWS/tunableop_results${r}.csv") - 5)) - log "freeze rank $r: $ROWS/tunableop_results${r}.csv ($n data rows)" - done - - # --- Update provenance.json --- - local prov=$ROWS/provenance.json - local config_list - config_list=$(echo "$ARMS" | tr ' ' ',' | sed 's/-m2//') - python3 - "$prov" "$ARMS" "$WORK" <<'PY' -import json, sys, datetime -from pathlib import Path -prov_path, arms, work = sys.argv[1], sys.argv[2].split(), sys.argv[3] -p = json.loads(Path(prov_path).read_text()) -new_keys = (Path(work) / "new_keys.txt").read_text().splitlines() if (Path(work) / "new_keys.txt").exists() else [] -curated_log = (Path(work) / "curate.log").read_text() if (Path(work) / "curate.log").exists() else "" -new_count = sum(1 for line in curated_log.splitlines() if "new " in line.lower() or "decisions:" in line.lower()) -p["campaign"] = { - "date": datetime.date.today().isoformat(), - "arms": arms, - "cells_per_arm": "c=1/c=8 x 1k/512 + 16k/1k", - "method": "record_untuned_enable during live boot + offline tune + curate", - "new_shapes_captured": len(new_keys), -} -p["validated"] = ( - datetime.date.today().isoformat() - + ": campaign expansion to " + " ".join(arms) - + " with the existing 719-row set as the source of truth; " - + str(len(new_keys)) + " new shape(s) captured, " - + "re-tuned at >=10 iters / 25 ms budget, curated against fresh same-process" - + " heuristic / current / new measurements (tools/rdna2_028/{tune_prod_shapes," - + "curate_tunableop_rows}.py), and merged into tunableop/rocblas-f30bb442e9b5/." - + " Solution IDs are build-specific." -) -Path(prov_path).write_text(json.dumps(p, indent=2) + "\n") -print("provenance updated", prov_path) -PY - log "provenance.json updated" - - echo "frozen" >"$WORK/freeze_status.txt" -} - -# --- Driver loop ----------------------------------------------------------- -sel_count() { timeout 8 sudo -n ipmitool sel list 2>/dev/null | grep -c 'PCI SERR'; } -SERR0=$(sel_count) -log "=== campaign 3b/c/d start: PHASE=$PHASE ARMS=$ARMS PCI-SERR=$SERR0 ===" -log "uptime=$(uptime -p) tree=$T rows=$ROWS" - -case "$PHASE" in - diff) diff_phase ;; - tune) diff_phase; tune_phase ;; - curate) diff_phase; tune_phase; curate_phase ;; - freeze) freeze_phase ;; - all) - diff_phase - tune_phase - curate_phase - freeze_phase - ;; - *) log "unknown PHASE=$PHASE"; exit 1 ;; -esac - -SERR1=$(sel_count) -log "=== done PHASE=$PHASE PCI-SERR=$SERR1 (base $SERR0) ===" -[ "$SERR1" -gt "$SERR0" ] && { echo "STOP NEW PCI SERR" >"$WORK/STOP"; log "STOP: new PCI SERR"; exit 2; } -exit 0 diff --git a/tools/rdna2_028/campaign_3d_freeze.sh b/tools/rdna2_028/campaign_3d_freeze.sh deleted file mode 100755 index 3319e9afc8a8..000000000000 --- a/tools/rdna2_028/campaign_3d_freeze.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# STEP 3 freeze + commit: take the curate decisions from one or more campaign -# arms and add the new rows to the existing tunableop/rocblas-f30bb442e9b5/ -# 4-rank rows. Existing rows are preserved (no drop); only the "new" decisions -# from the curate are appended. provenance.json is updated with the campaign -# metadata. Caller is responsible for git commit + push. -set -uo pipefail - -V=/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0 -T=/home/chenco_adm/vllm-rdna-0.28.0 -WORK=${WORK:-/home/chenco_adm/w4a8_runs/tunableop-campaign} -ROWS=$T/tunableop/rocblas-f30bb442e9b5 - -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$WORK/freeze.log"; } - -log "=== freeze: rows=$ROWS work=$WORK ===" - -TOTAL_NEW=0 -TOTAL_DUP=0 -for arm_dir in "$WORK"/curated_*; do - [ -d "$arm_dir" ] || continue - arm=$(basename "$arm_dir" | sed 's/^curated_//') - dec=$arm_dir/decisions.json - [ -f "$dec" ] || { log "no decisions.json in $arm_dir; skipping"; continue; } - python3 - <= 2 and r[0] == "Validator"] -existing_keys = set(r[1] for r in existing if len(r) >= 4 and r[1].startswith(("tn_", "nn_"))) -new_rows = [r for r in rows if len(r) >= 4 and r[1] in new_set] -added = [] -for r in new_rows: - if r[1] in existing_keys: - print(f" dup: {r[1]} (already in existing)") - continue - added.append(r) -print(f"arm: $arm new={len(new_keys)} dup={len(new_keys)-len(added)} added={len(added)}") -import os -os.makedirs(f"{arm_dir}/freeze", exist_ok=True) -for rank in (0, 1, 2, 3): - out = list(validators) - for r in existing: - if len(r) >= 4 and r[1].startswith(("tn_", "nn_")): - out.append(r) - out.extend(added) - with open(f"{arm_dir}/freeze/tunableop_results{rank}.csv", "w") as f: - w = csv.writer(f, lineterminator="\n") - w.writerows(out) - print(f" rank {rank}: {len(out) - len(validators)} data rows") -PY -done - -log "freezing the LAST arm's frozen rows into $ROWS (idempotent: existing rows preserved)" -LATEST=$(ls -d "$WORK"/curated_*/freeze 2>/dev/null | tail -1) -if [ -z "$LATEST" ]; then - log "ERROR: no freeze dir found"; exit 1 -fi -log "using $LATEST" -PRE_COUNT=$(grep -c "^GemmTunableOp" "$ROWS/tunableop_results0.csv") -for r in 0 1 2 3; do - cp "$LATEST/tunableop_results${r}.csv" "$ROWS/tunableop_results${r}.csv" - POST_COUNT=$(grep -c "^GemmTunableOp" "$ROWS/tunableop_results${r}.csv") - log "rank $r: $ROWS/tunableop_results${r}.csv ($POST_COUNT data rows, was $PRE_COUNT)" -done - -python3 - "$ROWS/provenance.json" "$WORK" <<'PY' -import json, datetime, os, sys -from pathlib import Path -prov_path, work = sys.argv[1], sys.argv[2] -p = json.loads(Path(prov_path).read_text()) - -arms = [] -for d in sorted(Path(work).glob("curated_*")): - if d.is_dir() and (d / "decisions.json").exists(): - arms.append(d.name.removeprefix("curated_")) -n_added = (Path(work) / "frozen_added.txt").read_text().strip() if (Path(work) / "frozen_added.txt").exists() else "n/a" - -p["campaign"] = { - "date": datetime.date.today().isoformat(), - "arms_covered": arms, - "method": "capture-via-record_untuned + offline tune (>=10 iters, 25ms budget) + per-arm curate (heuristic/current/new A/B) + additive freeze (existing rows preserved, new decisions appended)", - "rationale_for_additive": "The fresh A/B measurements under heavy GPU contention from the parallel campaign were noisy on the *existing* row set; 184 of the original 719 rows appeared heuristic-slower in the fresh measurement even though the prior curated set had validated them under low-contention. Conservative additive freeze keeps the 719 existing rows intact and appends only the NEW decisions the curate confirmed beat both heuristic and current by >= 3%.", - "rows_added": int(n_added) if n_added.isdigit() else None, -} - -existing_val = p.get("validated", "") -new_val = ( - datetime.date.today().isoformat() - + ": campaign expansion across arms " + ", ".join(arms) - + "; the curate's NEW decisions (each >= 3% faster than both heuristic and current) were appended; the existing 719 rows were preserved. " - + str(n_added) + " new data rows; provenance + storage unchanged." -) -p["validated"] = new_val -Path(prov_path).write_text(json.dumps(p, indent=2) + "\n") -print("provenance updated", prov_path) -PY - -log "freeze done. next: commit + push." diff --git a/tools/rdna2_028/campaign_matrix.sh b/tools/rdna2_028/campaign_matrix.sh deleted file mode 100644 index fbe4b120bc58..000000000000 --- a/tools/rdna2_028/campaign_matrix.sh +++ /dev/null @@ -1,256 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# Flash-Next TunableOp campaign driver: capture the observed GEMM shapes and -# validate the frozen rows across the serving matrix. -# -# MODE=capture boot with record_untuned_enable=1 pointed at a scratch file -# (PYTORCH_TUNABLEOP_RECORD_UNTUNED=1 + -# PYTORCH_TUNABLEOP_UNTUNED_FILENAME=/untuned.csv). -# NOTE: record mode DISABLES the results lookup, so the cells -# run on rocBLAS heuristics -- they are shape discovery, not -# performance validation. Writes shapes_.txt. -# MODE=validate boot with the fork rows via tunableop_env.sh (lookup-only, -# no record), run the measured cells; those ARE validation. -# Writes cells.csv + coherence + markers. -# -# Env: MODE, ARMS (space list), OUT, MODEL, CELLS, PORT base. -# No /tmp. Captures + logs live under /home/chenco_adm/w4a8_runs/. -set -uo pipefail - -MODE=${MODE:-capture} -V=/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0 -T=/home/chenco_adm/vllm-rdna-0.28.0 -MODEL=${MODEL:-/home/chenco_adm/hfcache/hub/models--wtdcode--Qwen3.8-Flash-Next-AWQ-W4A16/snapshots/0939125b929543a783ce700c90e36dd1a575c00c} -OUT=${OUT:-/home/chenco_adm/w4a8_runs} -CELLS=${CELLS:-"1 1024 512 221|8 1024 512 222|1 16384 1024 111|8 16384 1024 112"} -ARMS=${ARMS:-"w4a16-fa-m2"} -CAP_ROOT=$OUT/_captures -TRITON_TEMPLATE=${TRITON_TEMPLATE:-$T/cache/triton} - -ROCM_SDK_LIB=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -ROCM_SDK=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/rocm_sysdeps/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - -DRIVER_LOG=$OUT/campaign_${MODE}.log -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$DRIVER_LOG"; } - -our_pids() { - local p exe - for p in $(pgrep -f "entrypoints.cli.main serve|entrypoints.openai.api_server|VLLM::Worker|VLLM::EngineCore|PleOffloadWorker" 2>/dev/null); do - exe=$(readlink -f "/proc/$p/exe" 2>/dev/null || true) - case "$exe" in */venv-7.14.0_0.28.0/bin/python*) echo "$p" ;; esac - done -} - -hygiene() { - local n=0 left p - for p in $(our_pids); do kill -TERM "$p" 2>/dev/null && n=$((n + 1)); done - [ "$n" -gt 0 ] && sleep 8 - for p in $(our_pids); do kill -KILL "$p" 2>/dev/null; done - sleep 2 - left=$(our_pids | tr '\n' ' ') - log "hygiene: terminated=$n remaining_our=[${left% }]" - [ -n "${left// /}" ] && return 1 - return 0 -} - -sel_count() { timeout 10 sudo -n ipmitool sel list 2>/dev/null | grep -c 'PCI SERR'; } -sel_snap() { - local tag=$1 c - c=$(sel_count) - log "SEL[$tag] PCI-SERR=$c base=${SERR0:-?} uptime=$(uptime -p)" - if [ -n "${SERR0:-}" ] && [ "${c:-0}" -gt "$SERR0" ]; then - echo "STOP NEW PCI SERR after $tag ($c > $SERR0)" >"$OUT/STOP" - log "STOP: new PCI SERR after $tag"; return 1 - fi - return 0 -} - -teardown() { - local api p - api=$(grep -oE "APIServer pid=[0-9]+" "$D/serve.log" 2>/dev/null | head -1 | cut -d= -f2) - [ -n "$api" ] && { kill -TERM "$api" 2>/dev/null; sleep 10; } - for p in $(our_pids); do kill -TERM "$p" 2>/dev/null; done - sleep 6 - for p in $(our_pids); do kill -KILL "$p" 2>/dev/null; done - sleep 2 -} - -run_arm() { - local arm=$1 - case "$arm" in - w4a16-fa-m0) W4A8=0 MTP=0 ATTN=fa PORT=18300 ;; - w4a16-fa-m2) W4A8=0 MTP=2 ATTN=fa PORT=18301 ;; - w4a16-triton-m0) W4A8=0 MTP=0 ATTN=triton PORT=18302 ;; - w4a16-triton-m2) W4A8=0 MTP=2 ATTN=triton PORT=18303 ;; - w4a8-fa-m0) W4A8=1 MTP=0 ATTN=fa PORT=18304 ;; - w4a8-fa-m2) W4A8=1 MTP=2 ATTN=fa PORT=18305 ;; - w4a8-triton-m0) W4A8=1 MTP=0 ATTN=triton PORT=18306 ;; - w4a8-triton-m2) W4A8=1 MTP=2 ATTN=triton PORT=18307 ;; - *) log "unknown arm: $arm"; return 1 ;; - esac - TAG=${TAG_OVERRIDE:-2026-09-30_${MODE}_$arm} - local CACHE=$T/cache/arm-${MODE}-$arm - D=$OUT/$TAG - local CAP_DIR=$CAP_ROOT/$arm - mkdir -p "$D/cells" "$CAP_DIR" - - log "=== ARM=$arm MODE=$MODE W4A8=$W4A8 MTP=$MTP ATTN=$ATTN TAG=$TAG PORT=$PORT ===" - hygiene || { log "aborting: hygiene failed"; return 1; } - if ss -ltn 2>/dev/null | grep -q ":$PORT "; then log "aborting: port $PORT bound"; return 1; fi - rm -rf "$CACHE" - mkdir -p "$CACHE/inductor" "$CACHE/extensions" - if [ -d "$TRITON_TEMPLATE" ]; then - cp -a "$TRITON_TEMPLATE" "$CACHE/triton" - log "triton cache seeded ($(ls "$CACHE/triton" | wc -l) entries)" - else - mkdir -p "$CACHE/triton" - fi - rm -f "$CACHE/rdna_ar_wedged" - SERR0=$(sel_count) - log "arm cache=$CACHE baseline PCI-SERR=$SERR0" - sel_snap preflight || return 2 - - local EXTRA_ENV=() - if [ "$MODE" = "capture" ]; then - rm -f "$CAP_DIR"/untuned*.csv "$CAP_DIR"/untuned.csv - EXTRA_ENV=( - PYTORCH_TUNABLEOP_RECORD_UNTUNED=1 - PYTORCH_TUNABLEOP_UNTUNED_FILENAME="$CAP_DIR/untuned.csv" - PYTORCH_TUNABLEOP_TUNING=0 - ) - fi - - T0=$(date +%s) - setsid nohup env \ - VENV="$V" VLLM_TREE="$T" MTP="$MTP" ATTN="$ATTN" TP=4 PORT="$PORT" MODEL="$MODEL" \ - VLLM_CACHE_ROOT="$CACHE" TORCHINDUCTOR_CACHE_DIR="$CACHE/inductor" \ - TRITON_CACHE_DIR="$CACHE/triton" TORCH_EXTENSIONS_DIR="$CACHE/extensions" \ - VLLM_RDNA2_W4A8_SDOT4="$W4A8" \ - VLLM_RDNA_AR=1 VLLM_RDNA_AR_MAX_KB=64 VLLM_RDNA_AR_ONESHOT_KB=64 \ - VLLM_RDNA_AR_BLOCKS=0 VLLM_RDNA_AR_PACE=0 VLLM_FORCE_CUSTOM_ALL_REDUCE=0 \ - NCCL_P2P_LEVEL=pxb RCCL_P2P_NET_DISABLE=1 RCCL_P2P_BATCH_ENABLE=1 \ - NCCL_PROTO=Simple RCCL_MSCCL_ENABLE=0 PYTHONFAULTHANDLER=1 \ - HIP_VISIBLE_DEVICES=0,1,2,3 "${EXTRA_ENV[@]}" \ - bash "$T/tools/rdna/serve_gfx1030_flashnext_mtp.sh" >"$D/serve.log" 2>&1 /dev/null | grep -q flash-next && { ready=1; break; } - [ -f "$OUT/STOP" ] && break - if pgrep -f "[a]pi_server" >/dev/null; then dead=0; else dead=$((dead + 1)); fi - if [ "$dead" -ge 3 ]; then log "server gone during boot (after $((i*10))s)"; break; fi - done - local COLD=$(( $(date +%s) - T0 )) - if [ "$ready" != "1" ]; then - log "NOT READY after ${COLD}s"; tail -30 "$D/serve.log" | tee -a "$DRIVER_LOG" | tail -20 - teardown; hygiene; echo "NOT_READY" >"$D/status.txt"; return 1 - fi - log "READY t=${COLD}s"; echo "$COLD" >"$D/cold_compile_seconds.txt" - sel_snap after_boot || { teardown; hygiene; return 1; } - - # Warmup (throwaway) then coherence. - for wl in "1024 64 999" "16384 64 998"; do - set -- $wl - ( cd "$D" && "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model "$MODEL" --served-model-name flash-next --dataset-name random \ - --random-input-len "$1" --random-output-len "$2" --num-prompts 1 --max-concurrency 1 \ - --ignore-eos --request-rate inf --seed "$3" --temperature 0 \ - --save-result --result-dir "$D/warmup_$1" ) >"$D/warmup_$1.log" 2>&1 - done - log "warmup done" - "$V/bin/python" "$T/tools/rdna2_028/probe_w4a8.py" "http://127.0.0.1:$PORT/v1" flash-next 1 >"$D/coherence.txt" 2>&1 - log "coherence rc=$? $(grep -c 'OK ' "$D/coherence.txt")/$(grep -cE '^\[' "$D/coherence.txt") OK" - - # Measured cells. - local cell - IFS='|' read -ra _cells <<<"$CELLS" - for cell in "${_cells[@]}"; do - set -- $cell - local N=$1 IN=$2 OUTL=$3 SEED=$4 - local CD="$D/cells/c${IN}_${N}" - log "bench c=$N in=$IN start" - ( cd "$D" && "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model "$MODEL" --served-model-name flash-next --dataset-name random \ - --random-input-len "$IN" --random-output-len "$OUTL" --num-prompts "$N" --max-concurrency "$N" \ - --ignore-eos --request-rate inf --seed "$SEED" --temperature 0 \ - --save-result --result-dir "$CD" ) >"$CD.log" 2>&1 - log "bench c=$N in=$IN done: $(grep -m1 'Output token throughput' "$CD.log" | tr -s ' ')" - "$V/bin/python" - "$PORT" "$CD" <<'PY' >"$CD/coherence.txt" 2>&1 -import json, sys, urllib.request -port, out = sys.argv[1], sys.argv[2] -def ask(p, mt=16): - body = json.dumps({"model": "flash-next", "prompt": p, "max_tokens": mt, - "temperature": 0.0, "ignore_eos": True}).encode() - r = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions", data=body, - headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(r, timeout=600) as resp: - return json.load(resp)["choices"][0]["text"] or "" -for name, p, exp in [("france", "The capital of France is", "paris"), ("math", "2 + 2 =", "4")]: - try: t = ask(p) - except Exception as e: t = f"" - print(f"[{name}] coherent={exp in t.lower()} | {t[:120]!r}") -PY - sel_snap "after_c${N}_${IN}" || { teardown; hygiene; return 1; } - done - - # Markers. - { - echo "=== W4A8 MoE marker (expected iff W4A8=1) ===" - grep -m4 "W4A8\|w4a8\|sdot4" "$D/serve.log" | grep -v "W4A8-CALLTRACE" | head -8 || echo "(none)" - echo "=== TunableOp files ===" - grep -h "reading tuning results from\|TunableOp lookup\|record" "$D/serve.log" | sort -u | head -6 || echo "(none)" - echo "=== rdna_ar ===" - grep -h "rdna_ar:" "$D/serve.log" | sort -u | head -4 || echo "(none)" - echo "=== attention backend ===" - grep -h "RDNA_ATTN\|TRITON_ATTN\|attention backend" "$D/serve.log" | sort -u | head -4 || echo "(none)" - } >"$D/markers.txt" 2>&1 - tail -12 "$D/markers.txt" | tee -a "$DRIVER_LOG" - - # Mode-specific artifacts. - if [ "$MODE" = "capture" ]; then - local raw=0 - for f in "$CAP_DIR"/untuned[0-9].csv; do [ -f "$f" ] && raw=$((raw + $(wc -l <"$f"))); done - cat "$CAP_DIR"/untuned[0-9].csv 2>/dev/null | awk -F, '$1 ~ /^Gemm/ {print $1","$2}' | sort -u >"$CAP_DIR/shapes_$arm.txt" - cp "$CAP_DIR"/untuned[0-9].csv "$CAP_DIR/" 2>/dev/null || true - log "capture: raw lines=$raw unique keys=$(wc -l <"$CAP_DIR/shapes_$arm.txt") -> shapes_$arm.txt" - else - "$V/bin/python" "$T/tools/rdna2_028/flashnext_w4a8_extract.py" "$D" >"$D/cells.csv" 2>"$D/extract.err" || true - cat "$D/cells.csv" | tee -a "$DRIVER_LOG" | tail -6 - grep "SpecDecoding metrics" "$D/serve.log" >"$D/acceptance.txt" 2>/dev/null || true - fi - - teardown - hygiene - sel_snap after_teardown || true - log "done $arm (W4A8=$W4A8 MTP=$MTP ATTN=$ATTN)" - echo "OK" >"$D/status.txt" -} - -# --- Driver ---------------------------------------------------------------- -SERR0=$(sel_count) -log "=== campaign_${MODE} start ARMS=[$ARMS] PCI-SERR=$SERR0 uptime=$(uptime -p) ===" -hygiene || { log "hygiene failed at start"; exit 1; } - -rc=0 -for arm in $ARMS; do - log "---------------------------------------------------------------" - if ! run_arm "$arm"; then - log "ARM FAIL: $arm" - rc=1 - grep -q "PCI SERR" "$OUT/STOP" 2>/dev/null && { log "stopping: PCI SERR"; break; } - hygiene || { log "hygiene failed after $arm"; break; } - fi -done - -SERR1=$(sel_count) -log "=== campaign_${MODE} done rc=$rc PCI-SERR=$SERR1 (base $SERR0) ===" -[ "$SERR1" -gt "$SERR0" ] && { echo "STOP NEW PCI SERR" >"$OUT/STOP"; log "STOP: new PCI SERR"; exit 2; } -echo "DONE_$MODE rc=$rc" >"$OUT/campaign_${MODE}.status" -exit $rc diff --git a/tools/rdna2_028/exl3_analyze.py b/tools/rdna2_028/exl3_analyze.py deleted file mode 100644 index 8762f43d3801..000000000000 --- a/tools/rdna2_028/exl3_analyze.py +++ /dev/null @@ -1,28 +0,0 @@ -import csv, json -from pathlib import Path -W = Path("/home/chenco_adm/w4a8_runs/exl3-tunableop") -dec = json.load(open(W / "curated/decisions.json")) -src = set() -for r in csv.reader(open("/home/chenco_adm/vllm-rdna-0.28.0/tunableop/rocblas-f30bb442e9b5/tunableop_results0.csv")): - if len(r) >= 4 and r[0].startswith("GemmTunableOp"): - src.add(r[1]) -ed = [k for k, v in dec.items() if v == "dropped" and k in src] -print("EXISTING keys DROPPED:", len(ed)) -ba = {} -for r in csv.DictReader(open(W / "curated/before_after.csv")): - ba[r["shape"]] = r -for k in ed: - r = ba.get(k, {}) - print(" %s heur=%s cur=%s cur_solver=%s" % (k, r.get("heur_ms"), r.get("current_ms"), r.get("current_solver"))) -print() -cap = Path("/home/chenco_adm/w4a8_runs/_captures-exl3/exl3-m0/shapes_exl3-m0.txt") -novel = set(l.split(",")[1] for l in cap.read_text().splitlines() if l.startswith("GemmTunableOp_Half")) -adopted = sorted(k for k in dec if dec[k] == "new" and k in novel) -print("Adopted EXL3 novel keys (%d):" % len(adopted)) -for k in adopted: - r = ba.get(k, {}) - try: - g = float(r.get("new_ms")) / float(r.get("heur_ms")) - except Exception: - g = None - print(" %s heur=%s new=%s solver=%s speedup=%s" % (k, r.get("heur_ms"), r.get("new_ms"), r.get("new_solver"), g)) diff --git a/tools/rdna2_028/exl3_capture.sh b/tools/rdna2_028/exl3_capture.sh deleted file mode 100755 index 6cc50a7fa580..000000000000 --- a/tools/rdna2_028/exl3_capture.sh +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# EXL3 27B TunableOp shape-capture driver. -# -# Boots the EXL3 27B launcher (tools/rdna/serve_gfx1030_exl3_27b.sh) once per arm -# with the record-untuned census enabled: -# PYTORCH_TUNABLEOP_RECORD_UNTUNED=1 -# PYTORCH_TUNABLEOP_UNTUNED_FILENAME=/untuned.csv -# record_untuned DISABLES the results lookup, so the whole fp16 GEMM stream is -# recorded (not looked up) and the cells run on rocBLAS heuristics. This is -# shape discovery only -- a capture boot can never be a validation boot. -# -# MODE=capture (only mode). ARMS default "exl3-m0 exl3-m2". -# Cells: 1k/512 + 16k/1k at c=1 and c=8, short (64-token) outputs so the -# decode/verify shape families fire without paying long generation. -# -# No /tmp. GPUs default 4-7 (the Flash-Next co-tenant owns 0-3). -set -uo pipefail - -V=/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0 -T=/home/chenco_adm/vllm-rdna-0.28.0 -MODEL=${MODEL:-/home/chenco_adm/models/Qwen3.8-27B-exl3-3.00bpw} -OUT=${OUT:-/home/chenco_adm/w4a8_runs} -CAP_ROOT=${CAP_ROOT:-$OUT/_captures-exl3} -CELLS=${CELLS:-"1 1024 64 221|8 1024 64 222|1 16384 64 111|8 16384 64 112"} -WARM=${WARM:-"1024 32 999|16384 32 998"} -ARMS=${ARMS:-"exl3-m0 exl3-m2"} -GPUS=${GPUS:-4,5,6,7} - -ROCM_SDK_LIB=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -ROCM_SDK=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/rocm_sysdeps/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - -DRIVER_LOG=$OUT/exl3_capture.log -mkdir -p "$OUT" -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$DRIVER_LOG"; } - -# --- our EXL3 api_server / orphaned engine group only --------------------- -# A crashed api_server leaves orphaned VLLM::EngineCore + VLLM::Worker_TP* whose -# cmdline is just the setproctitle. Identify them by the stdout log path (this -# driver's run dir), which is unique to our captures; the co-tenant's log lives -# elsewhere and is never matched. -our_pids() { - local p fd - for p in $(pgrep -f "entrypoints.openai.api_server.*exl3-27b-mul1" 2>/dev/null); do echo "$p"; done - for p in $(pgrep -f "VLLM::EngineCore|VLLM::Worker_TP" 2>/dev/null); do - fd=$(readlink "/proc/$p/fd/1" 2>/dev/null || true) - case "$fd" in "$OUT"/2026-10-01_exl3-capture-*/serve.log) echo "$p" ;; esac - done -} -our_pgids() { - local p - for p in $(our_pids); do ps -o pgid= -p "$p" 2>/dev/null | tr -d ' '; done | sort -u -} -hygiene() { - local pg n=0 - for pg in $(our_pgids); do - kill -TERM "-$pg" 2>/dev/null && n=$((n + 1)) - done - [ "$n" -gt 0 ] && sleep 8 - for pg in $(our_pgids); do kill -KILL "-$pg" 2>/dev/null; done - sleep 2 - local left; left=$(our_pgids | tr '\n' ' ') - log "hygiene: terminated_groups=$n remaining=[${left% }]" - [ -n "${left// /}" ] && return 1 - return 0 -} -cleanup() { - local p - for p in $(our_pids); do kill -TERM "$p" 2>/dev/null; done - sleep 5 - for p in $(our_pids); do kill -KILL "$p" 2>/dev/null; done - return 0 -} -trap cleanup EXIT -sel_count() { timeout 8 sudo -n ipmitool sel list 2>/dev/null | grep -c 'PCI SERR'; } -sel_snap() { - local tag=$1 c; c=$(sel_count) - log "SEL[$tag] PCI-SERR=$c base=${SERR0:-?} uptime=$(uptime -p)" - if [ -n "${SERR0:-}" ] && [ "$c" -gt "$SERR0" ]; then - echo "STOP NEW PCI SERR after $tag ($c > $SERR0)" >"$OUT/STOP" - log "STOP: new PCI SERR after $tag"; return 1 - fi -} - -run_arm() { - local arm=$1 - case "$arm" in - exl3-m0) MTP=0 PORT=18400 TAG=2026-10-01_exl3-capture-m0 ;; - exl3-m2) MTP=2 PORT=18401 TAG=2026-10-01_exl3-capture-m2 ;; - *) log "unknown arm: $arm"; return 1 ;; - esac - local CACHE=$T/cache/arm-cap-$arm - local D=$OUT/$TAG - local CAP_DIR=$CAP_ROOT/$arm - mkdir -p "$D/cells" "$CAP_DIR" - - log "=== ARM=$arm MTP=$MTP TAG=$TAG PORT=$PORT ===" - hygiene || { log "aborting: hygiene failed"; return 1; } - if ss -ltn 2>/dev/null | grep -q ":$PORT "; then log "aborting: port $PORT bound"; return 1; fi - [ "${FRESH_CACHE:-0}" = "1" ] && rm -rf "$CACHE" - mkdir -p "$CACHE/inductor" "$CACHE/extensions" "$CACHE/triton" - rm -f "$CACHE/rdna_ar_wedged" - rm -f "$CAP_DIR"/untuned*.csv - SERR0=$(sel_count) - log "cache=$CACHE capture_dir=$CAP_DIR baseline PCI-SERR=$SERR0" - sel_snap preflight || return 2 - - T0=$(date +%s) - setsid nohup env \ - VENV="$V" VLLM_TREE="$T" MTP="$MTP" PORT="$PORT" MODEL="$MODEL" EAGER=0 \ - HIP_VISIBLE_DEVICES="$GPUS" \ - VLLM_CACHE_ROOT="$CACHE/vllm" TORCHINDUCTOR_CACHE_DIR="$CACHE/inductor" \ - TRITON_CACHE_DIR="$CACHE/triton" TORCH_EXTENSIONS_DIR="$CACHE/extensions" \ - TUNABLEOP=1 \ - PYTORCH_TUNABLEOP_TUNING=0 PYTORCH_TUNABLEOP_HIPBLASLT_ENABLED=0 \ - PYTORCH_TUNABLEOP_RECORD_UNTUNED=1 \ - PYTORCH_TUNABLEOP_UNTUNED_FILENAME="$CAP_DIR/untuned.csv" \ - PYTHONFAULTHANDLER=1 \ - bash "$T/tools/rdna/serve_gfx1030_exl3_27b.sh" >"$D/serve.log" 2>&1 /dev/null | grep -q exl3-27b-mul1 && { ready=1; break; } - [ -f "$OUT/STOP" ] && break - if [ -n "$(our_pids)" ]; then dead=0; else dead=$((dead + 1)); fi - if [ "$dead" -ge 3 ]; then log "server gone during boot (after $((i*10))s)"; break; fi - done - local COLD=$(( $(date +%s) - T0 )) - if [ "$ready" != "1" ]; then - log "NOT READY after ${COLD}s" - tail -40 "$D/serve.log" | tee -a "$DRIVER_LOG" - hygiene - echo "NOT_READY" >"$D/status.txt" - return 1 - fi - log "READY t=${COLD}s"; echo "$COLD" >"$D/cold_compile_seconds.txt" - sel_snap after_boot || { hygiene; return 1; } - - # Warmup (throwaway) then coherence (chat endpoint). - local wl - IFS='|' read -ra _warm <<<"$WARM" - for wl in "${_warm[@]}"; do - set -- $wl - log "warmup in=$1 out=$2" - ( cd "$D" && "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model "$MODEL" --served-model-name exl3-27b-mul1 --dataset-name random \ - --random-input-len "$1" --random-output-len "$2" --num-prompts 1 --max-concurrency 1 \ - --ignore-eos --request-rate inf --seed "$3" --temperature 0 \ - --save-result --result-dir "$D/warmup_$1" ) >"$D/warmup_$1.log" 2>&1 - done - log "warmup done" - "$V/bin/python" - "$PORT" >"$D/coherence.txt" 2>&1 <<'PY' -import json, sys, urllib.request -port = sys.argv[1] -def chat(prompt, mt=48): - body = json.dumps({"model": "exl3-27b-mul1", - "messages": [{"role": "user", "content": prompt}], - "max_tokens": mt, "temperature": 0.0}).encode() - r = urllib.request.Request(f"http://127.0.0.1:{port}/v1/chat/completions", data=body, - headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(r, timeout=600) as resp: - return json.load(resp)["choices"][0]["message"]["content"] or "" -ok = True -for name, prompt, expect in [("france", "The capital of France is", "paris"), - ("math", "2 + 2 =", "4")]: - try: - t = chat(prompt) - except Exception as e: - t = f"" - good = expect in t.lower() - ok = ok and good - print(f"[{name}] coherent={good} | {t[:120]!r}") -print("COHERENCE_RESULT:", "PASS" if ok else "FAIL") -PY - log "coherence rc=$? | $(grep -c 'coherent=True' "$D/coherence.txt")/2" - - # Measured (shape-firing) cells. - local cell - IFS='|' read -ra _cells <<<"$CELLS" - for cell in "${_cells[@]}"; do - set -- $cell - local N=$1 IN=$2 OUTL=$3 SEED=$4 - local CD="$D/cells/c${IN}_${N}" - log "bench c=$N in=$IN start" - ( cd "$D" && "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model "$MODEL" --served-model-name exl3-27b-mul1 --dataset-name random \ - --random-input-len "$IN" --random-output-len "$OUTL" --num-prompts "$N" --max-concurrency "$N" \ - --ignore-eos --request-rate inf --seed "$SEED" --temperature 0 \ - --save-result --result-dir "$CD" ) >"$CD.log" 2>&1 - log "bench c=$N in=$IN done: $(grep -m1 'Output token throughput' "$CD.log" | tr -s ' ')" - sel_snap "after_c${N}_${IN}" || { hygiene; return 1; } - done - - # Capture artefacts. - local raw=0 - for f in "$CAP_DIR"/untuned[0-9].csv; do [ -f "$f" ] && raw=$((raw + $(wc -l <"$f"))); done - cat "$CAP_DIR"/untuned[0-9].csv 2>/dev/null | awk -F, '$1 ~ /^Gemm/ {print $1","$2}' | sort -u >"$CAP_DIR/shapes_$arm.txt" - log "capture: raw lines=$raw unique keys=$(wc -l <"$CAP_DIR/shapes_$arm.txt") -> shapes_$arm.txt" - { - echo "=== TunableOp messages ===" - grep -h "TunableOp\|record_untuned\|tuning results" "$D/serve.log" | sort -u | head -10 || echo "(none)" - echo "=== attention backend ===" - grep -m2 "RDNA_ATTN\|attention backend" "$D/serve.log" | head -4 || echo "(none)" - echo "=== rdna_ar ===" - grep -h "rdna_ar:" "$D/serve.log" | sort -u | head -4 || echo "(none)" - } >"$D/markers.txt" 2>&1 - tail -6 "$D/markers.txt" | tee -a "$DRIVER_LOG" - - hygiene - sel_snap after_teardown || true - log "done $arm" - echo "OK" >"$D/status.txt" -} - -# --- driver --------------------------------------------------------------- -SERR0=$(sel_count) -log "=== exl3 capture start ARMS=[$ARMS] GPUs=$GPUS PCI-SERR=$SERR0 uptime=$(uptime -p) ===" -hygiene || { log "hygiene failed at start"; exit 1; } - -rc=0 -for arm in $ARMS; do - log "---------------------------------------------------------------" - if ! run_arm "$arm"; then - log "ARM FAIL: $arm (continuing)" - rc=1 - grep -q "PCI SERR" "$OUT/STOP" 2>/dev/null && { log "stopping: PCI SERR"; break; } - hygiene || { log "hygiene failed after $arm"; break; } - fi -done - -SERR1=$(sel_count) -log "=== exl3 capture done rc=$rc PCI-SERR=$SERR1 (base $SERR0) ===" -[ "$SERR1" -gt "$SERR0" ] && { echo "STOP NEW PCI SERR" >"$OUT/STOP"; log "STOP: new PCI SERR"; exit 2; } - -echo "=== capture summary ===" -for arm in $ARMS; do - f=$CAP_ROOT/$arm/shapes_$arm.txt - [ -f "$f" ] && echo "$arm: $(wc -l <"$f") unique Gemm keys" -done -exit $rc diff --git a/tools/rdna2_028/exl3_freezer.py b/tools/rdna2_028/exl3_freezer.py deleted file mode 100644 index 7b31e7ba96b9..000000000000 --- a/tools/rdna2_028/exl3_freezer.py +++ /dev/null @@ -1,62 +0,0 @@ -import csv, datetime, json, shutil, sys -from pathlib import Path - -T = Path(sys.argv[1]) -W = Path(sys.argv[2]) -ROWS = T / "tunableop/rocblas-f30bb442e9b5" -BK = W / "repo-rows-premerge" - -BK.mkdir(parents=True, exist_ok=True) -for f in list(ROWS.glob("tunableop_results*.csv")) + [ROWS / "provenance.json"]: - if f.exists(): - shutil.copy(f, BK / f.name) - -pre = sum(1 for l in (ROWS / "tunableop_results0.csv").read_text().splitlines() if l.startswith("GemmTunableOp")) -curated0 = (W / "curated/tunableop_results0.csv").read_text() -for r in range(4): - (ROWS / f"tunableop_results{r}.csv").write_text(curated0) -post = sum(1 for l in (ROWS / "tunableop_results0.csv").read_text().splitlines() if l.startswith("GemmTunableOp")) -print(f"FROZEN {pre} -> {post} rows/rank (backup at {BK})") - -def keys(path): - s = set() - for row in csv.reader(Path(path).open()): - if len(row) >= 4 and row[0].startswith("GemmTunableOp"): - s.add(row[1]) - return s - -prior = keys(BK / "tunableop_results0.csv") -adopted = (W / "adopted_new_keys.txt").read_text().splitlines() if (W / "adopted_new_keys.txt").exists() else [] -dropped = (W / "dropped_keys.txt").read_text().splitlines() if (W / "dropped_keys.txt").exists() else [] -existing_dropped = [k for k in dropped if k in prior] -new_keys = (W / "new_keys.txt").read_text().splitlines() if (W / "new_keys.txt").exists() else [] - -p = json.loads((ROWS / "provenance.json").read_text()) -p["rows_per_rank"] = post -p["campaign"] = { - "date": datetime.date.today().isoformat(), - "arm": "exl3-27b-mul1-m0 (Qwen3.8-27B-exl3-3.00bpw, mul1, TP=4, FA-RDNA2, FULL_AND_PIECEWISE)", - "capture_method": "tools/rdna2_028/exl3_capture.sh MODE=capture with PYTORCH_TUNABLEOP_RECORD_UNTUNED=1 + PYTORCH_TUNABLEOP_UNTUNED_FILENAME; record_untuned DISABLES the results lookup, so cells run on rocBLAS heuristics (shape discovery only).", - "captured_shapes": len(new_keys), - "mtp2_status": "NOT CAPTURED: EXL3 MTP head is not wired in this tree (qwen3_5_mtp.py registers only fc.weight for mtp.fc; the checkpoint's fc.suh/trellis/mul1 have nowhere to load -> ValueError at boot). Validated EXL3 serving config is MTP=0; MTP=2 needs an EXL3+MTP loader fix.", - "tune_method": "tools/rdna2_028/tune_prod_shapes.py --mode tune, novel keys only, 4 GPUs in parallel, >=10 iterations, 25 ms/solver, numerical check 0.01/0.01.", - "curate": { - "tool": "tools/rdna2_028/curate_tunableop_rows.py", - "adopt_margin": 0.03, - "drop_margin": 0.03, - "new": len(adopted), - "dropped": len(dropped), - "existing_dropped": len(existing_dropped), - "rows_before": pre, - "rows_after": post, - }, -} -p["validated"] = ( - datetime.date.today().isoformat() - + ": EXL3 27B expansion. " + str(len(adopted)) + " new rows adopted (21 novel EXL3 K=5120/1536/4352 shapes + 8 small-M decode extensions, each >=3% faster than both current and the rocBLAS heuristic; best 2.35x). " - + str(len(existing_dropped)) + " previously-shipped rows dropped (fresh measurement 4-7% slower than the heuristic). " - + "EXL3 27B rocBLAS fp16 GEMMs are K=5120/1536/4352 and collide with none of the Flash-Next K=2560/320/10240 set (0 collisions, 34 novel captured). " - + "MTP=2 not captured (EXL3 MTP head unwired). Lookup-hit proof 783/783. Prior set preserved at repo-rows-premerge/." -) -(ROWS / "provenance.json").write_text(json.dumps(p, indent=2) + "\n") -print(f"provenance updated: adopted={len(adopted)} dropped={len(dropped)} existing_dropped={len(existing_dropped)}") diff --git a/tools/rdna2_028/exl3_mtp2_validate.sh b/tools/rdna2_028/exl3_mtp2_validate.sh deleted file mode 100644 index ff5b9880809e..000000000000 --- a/tools/rdna2_028/exl3_mtp2_validate.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# EXL3 27B mul1 + MTP=2 validation driver: boot (eager or F&P), coherence, -# the four production cells, and SpecDecoding acceptance, with a PCI-SERR -# gate. One engine at a time, own per-run cache root. -# -# Usage: EAGER=1 TAG=... PORT=... bash exl3_mtp2_validate.sh -set -uo pipefail - -V=${VENV:-/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0} -T=${VLLM_TREE:-/home/chenco_adm/vllm-rdna-0.28.0} -OUT=${OUT:-/home/chenco_adm/w4a8_runs} -TAG=${TAG:-2026-10-01_exl3-27b-mtp2} -PORT=${PORT:-18107} -MTP=${MTP:-2} -EAGER=${EAGER:-0} -CG_MODE=${CG_MODE:-FULL_AND_PIECEWISE} -HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-4,5,6,7} -MODEL=${MODEL:-/home/chenco_adm/models/Qwen3.8-27B-exl3-3.00bpw} -TRITON_CACHE=${TRITON_CACHE:-$T/cache/triton} -CELLS=${CELLS:-"1 1024 512 301|8 1024 512 302|1 16384 1024 303|8 16384 1024 304"} -D=$OUT/$TAG -CACHE=$D/cache -mkdir -p "$D/cells" - -ROCM_SDK_LIB=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -ROCM_SDK=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/rocm_sysdeps/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$D/driver.log"; } -sel_count() { sudo -n ipmitool sel list 2>/dev/null | grep -c 'PCI SERR'; } - -our_pids() { - local p fd - for p in $(pgrep -f "entrypoints.openai.api_server.*exl3-27b-mul1" 2>/dev/null); do echo "$p"; done - for p in $(pgrep -f "VLLM::EngineCore|VLLM::Worker_TP" 2>/dev/null); do - fd=$(readlink "/proc/$p/fd/1" 2>/dev/null || true) - case "$fd" in "$D"/serve.log) echo "$p" ;; esac - done -} -our_pgids() { local p; for p in $(our_pids); do ps -o pgid= -p "$p" 2>/dev/null | tr -d ' '; done | sort -u; } -cleanup() { - local pg - for pg in $(our_pgids); do kill -TERM "-$pg" 2>/dev/null; done - sleep 6 - for pg in $(our_pgids); do kill -KILL "-$pg" 2>/dev/null; done - return 0 -} -trap cleanup EXIT - -log "=== EXL3 MTP=$MTP validate start (EAGER=$EAGER CG_MODE=$CG_MODE) tag=$TAG host=$(hostname) uptime=$(uptime -p) ===" -SERR0=$(sel_count); echo "$SERR0" >"$D/serr_baseline.txt"; log "PCI-SERR baseline=$SERR0" -cleanup -if ss -ltn 2>/dev/null | grep -q ":$PORT "; then log "port $PORT bound"; exit 1; fi - -setsid nohup env \ - VENV="$V" VLLM_TREE="$T" MODEL="$MODEL" MTP="$MTP" PORT="$PORT" EAGER="$EAGER" CG_MODE="$CG_MODE" \ - HIP_VISIBLE_DEVICES="$HIP_VISIBLE_DEVICES" \ - VLLM_CAUSAL_CONV1D_RDNA2_FWD=0 VLLM_CAUSAL_CONV1D_RDNA2_UPDATE=0 \ - VLLM_CACHE_ROOT="$CACHE/vllm" TORCHINDUCTOR_CACHE_DIR="$CACHE/inductor" \ - TRITON_CACHE_DIR="$TRITON_CACHE" TORCH_EXTENSIONS_DIR="$T/cache/extensions" \ - bash "$T/tools/rdna/serve_gfx1030_exl3_27b.sh" >"$D/serve.log" 2>&1 /dev/null | grep -q exl3-27b-mul1 && { ready=1; break; } - if [ -n "$(our_pids)" ]; then dead=0; else dead=$((dead + 1)); fi - [ "$dead" -ge 4 ] && { log "server died (dead=$dead)"; break; } -done -if [ "$ready" != "1" ]; then - log "NOT READY" - grep -a -i "fc.suh\|no module or parameter\|Traceback\|ValueError\|Error" "$D/serve.log" | tail -20 | tee -a "$D/driver.log" - exit 1 -fi -log "READY" -grep -h "TunableOp lookup enabled" "$D/serve.log" | head -1 | tee -a "$D/driver.log" -grep -ah "SpecDecoding metrics" "$D/serve.log" | wc -l | sed 's/^/specdecode_lines_at_ready=/' | tee -a "$D/driver.log" - -"$V/bin/python" - "$PORT" >"$D/coherence.txt" 2>&1 <<'PY' -import json, sys, urllib.request -port = sys.argv[1] -def chat(p, mt=64): - b = json.dumps({"model": "exl3-27b-mul1", "messages": [{"role": "user", "content": p}], - "max_tokens": mt, "temperature": 0.0}).encode() - r = urllib.request.Request(f"http://127.0.0.1:{port}/v1/chat/completions", data=b, - headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(r, timeout=600) as resp: - return json.load(resp)["choices"][0]["message"]["content"] or "" -ok = True -for name, p, exp in [("france", "The capital of France is", "paris"), ("math", "2 + 2 =", "4")]: - try: t = chat(p) - except Exception as e: t = f"" - good = exp in t.lower(); ok = ok and good - print(f"[{name}] coherent={good} | {t[:150]!r}") -print("COHERENCE_RESULT:", "PASS" if ok else "FAIL") -PY -log "coherence | $(grep -c 'coherent=True' "$D/coherence.txt")/2" - -log "warmup c=1 in=1024 out=128" -( cd "$D" && "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model "$MODEL" --served-model-name exl3-27b-mul1 \ - --dataset-name random --random-input-len 1024 --random-output-len 128 \ - --num-prompts 1 --max-concurrency 1 --ignore-eos --request-rate inf \ - --seed 900 --temperature 0 ) >"$D/warmup.log" 2>&1 - -IFS='|' read -ra _cells <<<"$CELLS" -for cell in "${_cells[@]}"; do - set -- $cell - N=$1; IN=$2; OUTL=$3; SEED=$4 - CD="$D/cells/c${IN}_${N}" - log "bench c=$N in=$IN" - before=$(grep -ac "SpecDecoding metrics" "$D/serve.log" || true) - ( cd "$D" && "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model "$MODEL" --served-model-name exl3-27b-mul1 \ - --dataset-name random --random-input-len "$IN" --random-output-len "$OUTL" \ - --num-prompts "$N" --max-concurrency "$N" --ignore-eos --request-rate inf \ - --seed "$SEED" --temperature 0 \ - --save-result --result-dir "$CD" ) >"$CD.log" 2>&1 - log " c=$N in=$IN done: $(grep -m1 'Output token throughput' "$CD.log" | tr -s ' ')" - sleep 12 - grep -a "SpecDecoding metrics" "$D/serve.log" | tail -n +$((before + 1)) >"$CD.acceptance.txt" - tail -1 "$CD.acceptance.txt" | tee -a "$D/driver.log" - c=$(sel_count); log " SEL PCI-SERR=$c" - if [ "$c" -gt "$SERR0" ]; then - log "STOP new PCI SERR ($c > $SERR0)" - echo STOP >"$D/STOP" - break - fi -done - -grep -ah "SpecDecoding metrics" "$D/serve.log" >"$D/acceptance.txt" -log "=== EXL3 MTP=$MTP validate done ===" -echo "DONE" >"$D/status.txt" diff --git a/tools/rdna2_028/exl3_tunableop_campaign.sh b/tools/rdna2_028/exl3_tunableop_campaign.sh deleted file mode 100755 index 76a7873bebf2..000000000000 --- a/tools/rdna2_028/exl3_tunableop_campaign.sh +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# EXL3 27B TunableOp campaign: turn the record-untuned census into a curated -# union row set and freeze it. -# -# PHASE=diff union the captured shapes, diff against the frozen rows, -# write novel GemmTunableOp_Half keys -# PHASE=tune offline-tune ONLY the novel keys (4 GPUs in parallel) -# PHASE=measure heuristic / current / new per shape over the union set -# PHASE=curate adopt >=3% / drop >3% / fold Default (curate_tunableop_rows.py) -# PHASE=freeze backup the prior rows + install the curated union + provenance -# PHASE=verify lookup-hit proof over the frozen rows -# PHASE=all diff -> tune -> measure -> curate -# -# GPUs: default 4-7 (HIP_VISIBLE_DEVICES=4,5,6,7 -> in-process 0..3). The -# Flash-Next co-tenant owns 0-3 and is never touched. No /tmp. -set -uo pipefail - -V=${VENV:-/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0} -T=${VLLM_TREE:-/home/chenco_adm/vllm-rdna-0.28.0} -WORK=${WORK:-/home/chenco_adm/w4a8_runs/exl3-tunableop} -ROWS=${ROWS:-$T/tunableop/rocblas-f30bb442e9b5} -CAP_ROOT=${CAP_ROOT:-/home/chenco_adm/w4a8_runs/_captures-exl3} -ARMS=${ARMS:-"exl3-m0 exl3-m2"} -GPUS=${GPUS:-4,5,6,7} -PHASE=${PHASE:-all} - -mkdir -p "$WORK" -ROCM_SDK_LIB=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -ROCM_SDK=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/rocm_sysdeps/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -export PYTORCH_TUNABLEOP_HIPBLASLT_ENABLED=0 TORCH_BLAS_PREFER_HIPBLASLT=0 - -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$WORK/driver.log"; } -sel_count() { timeout 8 sudo -n ipmitool sel list 2>/dev/null | grep -c 'PCI SERR'; } - -diff_phase() { - log "=== PHASE=diff ===" - : >"$WORK/cap_keys.txt" - for arm in $ARMS; do - f=$CAP_ROOT/$arm/shapes_$arm.txt - [ -f "$f" ] || { log "$arm: no shapes file ($f)"; continue; } - grep -E "^GemmTunableOp_Half_(TN|NN)," "$f" >>"$WORK/cap_keys.txt" - log "$arm: $(grep -cE '^GemmTunableOp_Half' "$f") GemmTunableOp_Half keys" - done - sort -u "$WORK/cap_keys.txt" -o "$WORK/cap_keys.txt" - awk -F, '$1 ~ /^GemmTunableOp/ {print $1","$2}' "$ROWS/tunableop_results0.csv" | sort -u >"$WORK/src_keys.txt" - comm -23 "$WORK/cap_keys.txt" "$WORK/src_keys.txt" >"$WORK/new_keys.txt" - comm -12 "$WORK/cap_keys.txt" "$WORK/src_keys.txt" >"$WORK/collide_keys.txt" - local n_cap n_src n_new n_col - n_cap=$(wc -l <"$WORK/cap_keys.txt"); n_src=$(wc -l <"$WORK/src_keys.txt") - n_new=$(wc -l <"$WORK/new_keys.txt"); n_col=$(wc -l <"$WORK/collide_keys.txt") - log "captured GemmTunableOp_Half=$n_cap current=$n_src colliding=$n_col NOVEL=$n_new" - [ "$n_new" -gt 0 ] && { log "novel keys:"; head -30 "$WORK/new_keys.txt" | tee -a "$WORK/driver.log"; [ "$n_new" -gt 30 ] && log "... ($((n_new - 30)) more)"; } -} - -build_tune_src() { - echo "Validator,PT_VERSION,2.12.0" >"$WORK/tune_source.csv" - echo "Validator,HIP_VERSION,714" >>"$WORK/tune_source.csv" - while IFS= read -r line; do - [ -z "$line" ] && continue - printf '%s,Default,0.001\n' "$line" >>"$WORK/tune_source.csv" - done <"$WORK/new_keys.txt" -} - -build_meas_src() { - cat "$ROWS/tunableop_results0.csv" >"$WORK/measure_source.csv" - while IFS= read -r line; do - [ -z "$line" ] && continue - printf '%s,Default,0.001\n' "$line" >>"$WORK/measure_source.csv" - done <"$WORK/new_keys.txt" -} - -tune_phase() { - local n_new; n_new=$(wc -l <"$WORK/new_keys.txt" 2>/dev/null || echo 0) - if [ "$n_new" -eq 0 ]; then log "=== PHASE=tune SKIPPED (no novel keys) ==="; return 0; fi - log "=== PHASE=tune ($n_new novel keys) ===" - build_tune_src - mkdir -p "$WORK/scratch" - local pids=() gpu - for gpu in 0 1 2 3; do - HIP_VISIBLE_DEVICES="$GPUS" "$V/bin/python" "$T/tools/rdna2_028/tune_prod_shapes.py" \ - --shape-source "$WORK/tune_source.csv" --mode tune --device "$gpu" \ - --scratch "$WORK/scratch" --rank "$gpu" \ - --iterations 10 --max-ms 25 \ - >"$WORK/tune_inproc$gpu.json" 2>"$WORK/tune_inproc$gpu.err" & - pids+=($!) - done - local rc=0 i - for i in 0 1 2 3; do - wait "${pids[$i]}" || { log "TUNE FAIL inproc=$i"; tail -15 "$WORK/tune_inproc$i.err" | tee -a "$WORK/driver.log"; rc=1; } - done - [ "$rc" -ne 0 ] && return 1 - for gpu in 0 1 2 3; do - log "tune rank $gpu rows=$(grep -c '^GemmTunableOp' "$WORK/scratch/tunableop_results$gpu.csv" 2>/dev/null || echo 0)" - done -} - -measure_phase() { - local n_new; n_new=$(wc -l <"$WORK/new_keys.txt" 2>/dev/null || echo 0) - if [ "$n_new" -eq 0 ]; then log "=== PHASE=measure SKIPPED (no novel keys) ==="; return 0; fi - log "=== PHASE=measure (union $(grep -c '^GemmTunableOp' "$WORK/measure_source.csv" 2>/dev/null || echo 0) shapes) ===" - build_meas_src - mkdir -p "$WORK/measure" - local cond - for cond in heuristic current new; do - local src=() - case $cond in - heuristic) src=(--heuristic) ;; - current) src=(--rows "$ROWS/tunableop_results0.csv") ;; - new) src=(--rows "$WORK/scratch/tunableop_results0.csv") ;; - esac - log "measure $cond" - HIP_VISIBLE_DEVICES="$GPUS" "$V/bin/python" "$T/tools/rdna2_028/tune_prod_shapes.py" \ - --shape-source "$WORK/measure_source.csv" --mode measure --device 0 "${src[@]}" \ - --warmup 5 --reps 15 --out "$WORK/measure/$cond.json" \ - >"$WORK/measure_$cond.out" 2>"$WORK/measure_$cond.err" \ - || { log "MEASURE FAIL $cond"; tail -15 "$WORK/measure_$cond.err" | tee -a "$WORK/driver.log"; return 1; } - done -} - -curate_phase() { - local n_new; n_new=$(wc -l <"$WORK/new_keys.txt" 2>/dev/null || echo 0) - if [ "$n_new" -eq 0 ]; then log "=== PHASE=curate SKIPPED (no novel keys) ==="; return 0; fi - log "=== PHASE=curate ===" - "$V/bin/python" "$T/tools/rdna2_028/curate_tunableop_rows.py" \ - --current-rows "$ROWS" --scratch-rows "$WORK/scratch" \ - --heur "$WORK/measure/heuristic.json" --cur "$WORK/measure/current.json" --new "$WORK/measure/new.json" \ - --adopt-margin 0.03 --drop-margin 0.03 --ranks 0,1,2,3 \ - --out "$WORK/curated" >"$WORK/curate.log" 2>&1 \ - || { log "CURATE FAIL"; cat "$WORK/curate.log" | tee -a "$WORK/driver.log"; return 1; } - grep -E "^decisions" "$WORK/curate.log" | tee -a "$WORK/driver.log" - # summary counts - python3 - "$WORK/curated/decisions.json" "$WORK" <<'PY' | tee -a "$WORK/driver.log" -import json, sys -from pathlib import Path -d = json.load(open(sys.argv[1])) -w = Path(sys.argv[2]) -rows0 = [l for l in (w/"curated/tunableop_results0.csv").read_text().splitlines() if l.startswith("GemmTunableOp")] -new = sorted(k for k, v in d.items() if v == "new") -dropped = sorted(k for k, v in d.items() if v == "dropped") -(w/"adopted_new_keys.txt").write_text("\n".join(new) + ("\n" if new else "")) -(w/"dropped_keys.txt").write_text("\n".join(dropped) + ("\n" if dropped else "")) -print(f"curated rows/rank={len(rows0)} adopted_new={len(new)} dropped={len(dropped)}") -PY -} - -freeze_phase() { - log "=== PHASE=freeze ===" - local pre=$WORK/repo-rows-premerge - mkdir -p "$pre" - cp "$ROWS"/tunableop_results*.csv "$ROWS/provenance.json" "$pre/" 2>/dev/null - local pre_n; pre_n=$(grep -c '^GemmTunableOp' "$ROWS/tunableop_results0.csv") - local r - for r in 0 1 2 3; do - cp "$WORK/curated/tunableop_results$r.csv" "$ROWS/tunableop_results$r.csv" - log "rank $r: $(grep -c '^GemmTunableOp' "$ROWS/tunableop_results$r.csv") rows (was $pre_n)" - done - local post_n; post_n=$(grep -c '^GemmTunableOp' "$ROWS/tunableop_results0.csv") - log "FROZEN $pre_n -> $post_n rows/rank (backup $pre)" -} - -verify_phase() { - log "=== PHASE=verify ===" - HIP_VISIBLE_DEVICES="$GPUS" "$V/bin/python" "$T/tools/rdna2_028/verify_tunableop_lookup.py" \ - --rows "$ROWS/tunableop_results0.csv" --out "$WORK/lookup_hits.json" --device 0 \ - >"$WORK/lookup_hit.out" 2>"$WORK/lookup_hit.err" \ - || { log "VERIFY FAIL"; tail -20 "$WORK/lookup_hit.err" | tee -a "$WORK/driver.log"; return 1; } - grep -E "^lookup hits|MISS" "$WORK/lookup_hit.out" | tee -a "$WORK/driver.log" -} - -SERR0=$(sel_count) -log "=== exl3 tunableop campaign PHASE=$PHASE ARMS=[$ARMS] GPUs=$GPUS PCI-SERR=$SERR0 ===" -log "tree=$T rows=$ROWS work=$WORK uptime=$(uptime -p)" - -case "$PHASE" in - diff) diff_phase ;; - tune) diff_phase; tune_phase ;; - measure) diff_phase; tune_phase; measure_phase; curate_phase ;; - curate) diff_phase; tune_phase; measure_phase; curate_phase ;; - freeze) freeze_phase ;; - verify) verify_phase ;; - all) diff_phase; tune_phase; measure_phase; curate_phase ;; - *) log "unknown PHASE=$PHASE"; exit 1 ;; -esac - -SERR1=$(sel_count) -log "=== done PHASE=$PHASE PCI-SERR=$SERR1 (base $SERR0) ===" -[ "$SERR1" -gt "$SERR0" ] && { echo "STOP NEW PCI SERR" >"$WORK/STOP"; log "STOP: new PCI SERR"; exit 2; } -exit 0 diff --git a/tools/rdna2_028/exl3_validate.sh b/tools/rdna2_028/exl3_validate.sh deleted file mode 100644 index 92a3e8b1b1fe..000000000000 --- a/tools/rdna2_028/exl3_validate.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# EXL3 27B e2e validation with the frozen TunableOp rows: boot lookup-only, -# prove the frozen rows are consumed, run a measured cell, check coherence. -# EXL3_VALIDATE=1 -# Reuses the warm capture cache so no recompile; the TunableOp env differs -# (lookup-only vs record) so the served numbers are the real frozen-row run. -set -uo pipefail -V=/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0 -T=/home/chenco_adm/vllm-rdna-0.28.0 -OUT=${OUT:-/home/chenco_adm/w4a8_runs} -TAG=${TAG:-2026-10-01_exl3-validate} -PORT=${PORT:-18106} -CACHE=$T/cache/arm-cap-exl3-m0 -D=$OUT/$TAG -CELLS=${CELLS:-"1 1024 512 301|8 1024 512 302|1 16384 1024 303"} -mkdir -p "$D/cells" - -ROCM_SDK_LIB=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -ROCM_SDK=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/rocm_sysdeps/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$OUT/exl3_validate.log"; } - -our_pids() { - local p fd - for p in $(pgrep -f "entrypoints.openai.api_server.*exl3-27b-mul1" 2>/dev/null); do echo "$p"; done - for p in $(pgrep -f "VLLM::EngineCore|VLLM::Worker_TP" 2>/dev/null); do - fd=$(readlink "/proc/$p/fd/1" 2>/dev/null || true) - case "$fd" in "$D"/serve.log) echo "$p" ;; esac - done -} -our_pgids() { local p; for p in $(our_pids); do ps -o pgid= -p "$p" 2>/dev/null | tr -d ' '; done | sort -u; } -cleanup() { - local pg - for pg in $(our_pgids); do kill -TERM "-$pg" 2>/dev/null; done - sleep 6 - for pg in $(our_pgids); do kill -KILL "-$pg" 2>/dev/null; done - return 0 -} -trap cleanup EXIT - -log "=== EXL3 validate start (frozen rows) tag=$TAG ===" -cleanup -if ss -ltn 2>/dev/null | grep -q ":$PORT "; then log "port $PORT bound"; exit 1; fi - -setsid nohup env \ - VENV="$V" VLLM_TREE="$T" MTP=0 PORT="$PORT" EAGER=0 TUNABLEOP=1 \ - HIP_VISIBLE_DEVICES=4,5,6,7 \ - VLLM_CACHE_ROOT="$CACHE/vllm" TORCHINDUCTOR_CACHE_DIR="$CACHE/inductor" \ - TRITON_CACHE_DIR="$CACHE/triton" TORCH_EXTENSIONS_DIR="$CACHE/extensions" \ - bash "$T/tools/rdna/serve_gfx1030_exl3_27b.sh" >"$D/serve.log" 2>&1 /dev/null | grep -q exl3-27b-mul1 && { ready=1; break; } - if [ -n "$(our_pids)" ]; then dead=0; else dead=$((dead + 1)); fi - [ "$dead" -ge 4 ] && { log "server died (dead=$dead)"; break; } -done -[ "$ready" != "1" ] && { log "NOT READY"; tail -30 "$D/serve.log"; exit 1; } -log "READY" -grep -h "TunableOp lookup enabled" "$D/serve.log" | head -1 | tee -a "$OUT/exl3_validate.log" -grep -h "RDNA_ATTN\|attention backend" "$D/serve.log" | head -2 | tee -a "$OUT/exl3_validate.log" - -# coherence -"$V/bin/python" - "$PORT" >"$D/coherence.txt" 2>&1 <<'PY' -import json, sys, urllib.request -port = sys.argv[1] -def chat(p, mt=48): - b = json.dumps({"model": "exl3-27b-mul1", "messages": [{"role": "user", "content": p}], - "max_tokens": mt, "temperature": 0.0}).encode() - r = urllib.request.Request(f"http://127.0.0.1:{port}/v1/chat/completions", data=b, - headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(r, timeout=600) as resp: - return json.load(resp)["choices"][0]["message"]["content"] or "" -ok = True -for name, p, exp in [("france", "The capital of France is", "paris"), ("math", "2 + 2 =", "4")]: - try: t = chat(p) - except Exception as e: t = f"" - good = exp in t.lower(); ok = ok and good - print(f"[{name}] coherent={good} | {t[:120]!r}") -print("COHERENCE_RESULT:", "PASS" if ok else "FAIL") -PY -log "coherence rc=$? | $(grep -c 'coherent=True' "$D/coherence.txt")/2" - -# measured cells -local_cell="" -IFS='|' read -ra _cells <<<"$CELLS" -for cell in "${_cells[@]}"; do - set -- $cell - N=$1; IN=$2; OUTL=$3; SEED=$4 - CD="$D/cells/c${IN}_${N}" - log "bench c=$N in=$IN" - ( cd "$D" && "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model /home/chenco_adm/models/Qwen3.8-27B-exl3-3.00bpw --served-model-name exl3-27b-mul1 \ - --dataset-name random --random-input-len "$IN" --random-output-len "$OUTL" \ - --num-prompts "$N" --max-concurrency "$N" --ignore-eos --request-rate inf --seed "$SEED" --temperature 0 \ - --save-result --result-dir "$CD" ) >"$CD.log" 2>&1 - log " c=$N in=$IN done: $(grep -m1 'Output token throughput' "$CD.log" | tr -s ' ')" -done - -log "=== EXL3 validate done ===" -echo "DONE" >"$D/status.txt" diff --git a/tools/rdna2_028/flashnext_w4a8_arm.sh b/tools/rdna2_028/flashnext_w4a8_arm.sh deleted file mode 100644 index c53cbf625114..000000000000 --- a/tools/rdna2_028/flashnext_w4a8_arm.sh +++ /dev/null @@ -1,258 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# ONE arm of the Qwen3.8-Flash-Next W4A8/W4A16 matrix on gfx1030. -# -# Launcher: tools/rdna/serve_gfx1030_flashnext_mtp.sh (the documented Flash-Next -# MTP path: V2 runner, FA-RDNA2, MTP ladder [3,6,12,24] + local argmax, RDNA AR -# one-shot <=64 KiB, prefix caching, FULL_AND_PIECEWISE, TP=4/EP4, PLE offload). -# This driver adds the one thing the launcher does not own: an arm-tagged, -# isolated graph/Triton cache + the W4A8 opt-in env + measured cells. -# -# Env: -# ARM=w4a8on|w4a8off cache suffix (arm dir = $T/cache/arm-fn-$ARM) -# W4A8=1|0 VLLM_RDNA2_W4A8_SDOT4 -# MTP=0|2 speculative tokens (launcher builds the capture ladder) -# TAG=... output dir under $OUT -# PORT, SERR_BASE, OUT, MODEL, CELLS, WARM_TRITON, DO_WARMUP -# -# Guards: hygiene scoped to the 0.28.0 venv only (never a co-tenant on -# venv-7.14.0), IPMI PCI-SERR baseline before/after every phase, stop marker. -# No /tmp: everything under /home/chenco_adm/w4a8_runs//. -set -uo pipefail - -V=/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0 -T=/home/chenco_adm/vllm-rdna-0.28.0 -MODEL=${MODEL:-/home/chenco_adm/hfcache/hub/models--wtdcode--Qwen3.8-Flash-Next-AWQ-W4A16/snapshots/0939125b929543a783ce700c90e36dd1a575c00c} -ARM=${ARM:?set ARM (w4a8on|w4a8off)} -W4A8=${W4A8:?set W4A8 (1|0)} -MTP=${MTP:-0} -TAG=${TAG:-2026-09-29_fn-$ARM-m$MTP} -PORT=${PORT:-18260} -OUT=${OUT:-/home/chenco_adm/w4a8_runs} -CACHE=$T/cache/arm-fn-$ARM -TRITON_TEMPLATE=${TRITON_TEMPLATE:-$T/cache/triton} -WARM_TRITON=${WARM_TRITON:-1} -DO_WARMUP=${DO_WARMUP:-1} -CELLS=${CELLS:-"1 1024 512 221|8 1024 512 222|1 16384 1024 111|8 16384 1024 112"} -D=$OUT/$TAG -mkdir -p "$D/cells" - -ROCM_SDK_LIB=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -ROCM_SDK=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/rocm_sysdeps/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$D/driver.log"; } - -# --- OUR processes only: the 0.28.0 venv, never the co-tenant's venv-7.14.0 --- -our_pids() { - local p exe - for p in $(pgrep -f "entrypoints.cli.main serve|entrypoints.openai.api_server|VLLM::Worker|VLLM::EngineCore|PleOffloadWorker" 2>/dev/null); do - exe=$(readlink -f "/proc/$p/exe" 2>/dev/null || true) - case "$exe" in - */venv-7.14.0_0.28.0/bin/python*) echo "$p" ;; - esac - done -} - -show_pids() { - local p exe cwd - for p in $(our_pids); do - exe=$(readlink -f "/proc/$p/exe" 2>/dev/null) - cwd=$(readlink -f "/proc/$p/cwd" 2>/dev/null) - echo " pid=$p exe=$exe cwd=$cwd cmd=$(tr '\0' ' ' /dev/null | cut -c1-70)" - done -} - -hygiene() { - local n=0 p left - show_pids | sed 's/^/hyg:before /' | tee -a "$D/driver.log" - for p in $(our_pids); do kill -TERM "$p" 2>/dev/null && n=$((n + 1)); done - [ "$n" -gt 0 ] && sleep 8 - for p in $(our_pids); do kill -KILL "$p" 2>/dev/null; done - sleep 2 - left=$(our_pids | tr '\n' ' ') - log "hygiene: terminated=$n remaining_our=[${left% }]" - { - echo "=== [$(date -Is)] KFD map ===" - sudo -n rocm-smi --showpids 2>/dev/null | grep -E "^\s+[0-9]+" || true - } >>"$D/kfd.log" - [ -n "${left// /}" ] && { log "HYGIENE FAIL: $left"; return 1; } - return 0 -} - -sel_count() { sudo -n ipmitool sel list 2>/dev/null | grep -c 'PCI SERR'; } -sel_snap() { - local tag=$1 c - c=$(sel_count) - log "SEL[$tag] PCI-SERR=$c base=${SERR0:-?} uptime=$(uptime -p)" - if [ -n "${SERR0:-}" ] && [ "${c:-0}" -gt "$SERR0" ]; then - echo "STOP NEW PCI SERR after $tag ($c > $SERR0)" >"$D/STOP" - log "STOP: new PCI SERR after $tag" - return 1 - fi - return 0 -} - -teardown() { - local api p - api=$(grep -oE "APIServer pid=[0-9]+" "$D/serve.log" 2>/dev/null | head -1 | cut -d= -f2) - [ -n "$api" ] && kill -TERM "$api" 2>/dev/null && sleep 10 - for p in $(our_pids); do kill -TERM "$p" 2>/dev/null; done - sleep 6 - for p in $(our_pids); do kill -KILL "$p" 2>/dev/null; done - sleep 2 -} - -# --- Preflight ------------------------------------------------------------- -log "=== ARM=$ARM W4A8=$W4A8 MTP=$MTP TAG=$TAG PORT=$PORT RDNA_AR=1 (one-shot 64K) ===" -log "preflight uptime=$(uptime -p) host=$(hostname)" -hygiene || { log "aborting: hygiene failed"; exit 1; } -if ss -ltn 2>/dev/null | grep -q ":$PORT "; then log "aborting: port $PORT bound"; exit 1; fi -rm -rf "$CACHE" -mkdir -p "$CACHE/inductor" "$CACHE/extensions" -if [ "$WARM_TRITON" = "1" ] && [ -d "$TRITON_TEMPLATE" ]; then - cp -a "$TRITON_TEMPLATE" "$CACHE/triton" - log "triton cache seeded from $TRITON_TEMPLATE ($(ls "$CACHE/triton" | wc -l) entries)" -else - mkdir -p "$CACHE/triton"; log "triton cache FRESH (cold autotune will run)" -fi -rm -f "$CACHE/rdna_ar_wedged" -[ -n "${SERR_BASE:-}" ] && SERR0=$SERR_BASE || SERR0=$(sel_count) -log "arm cache=$CACHE baseline PCI-SERR=$SERR0" -sel_snap preflight || exit 1 - -# --- Launch ---------------------------------------------------------------- -T0=$(date +%s) -setsid nohup env \ - VENV="$V" VLLM_TREE="$T" MTP="$MTP" ATTN=fa TP=4 PORT="$PORT" MODEL="$MODEL" \ - VLLM_CACHE_ROOT="$CACHE" TORCHINDUCTOR_CACHE_DIR="$CACHE/inductor" \ - TRITON_CACHE_DIR="$CACHE/triton" TORCH_EXTENSIONS_DIR="$CACHE/extensions" \ - VLLM_RDNA2_W4A8_SDOT4="$W4A8" \ - VLLM_RDNA_AR=1 VLLM_RDNA_AR_MAX_KB=64 VLLM_RDNA_AR_ONESHOT_KB=64 \ - VLLM_RDNA_AR_BLOCKS=0 VLLM_RDNA_AR_PACE=0 VLLM_FORCE_CUSTOM_ALL_REDUCE=0 \ - NCCL_P2P_LEVEL=pxb RCCL_P2P_NET_DISABLE=1 RCCL_P2P_BATCH_ENABLE=1 \ - NCCL_PROTO=Simple RCCL_MSCCL_ENABLE=0 PYTHONFAULTHANDLER=1 \ - HIP_VISIBLE_DEVICES=0,1,2,3 \ - bash "$T/tools/rdna/serve_gfx1030_flashnext_mtp.sh" >"$D/serve.log" 2>&1 /dev/null | grep -q flash-next && { ready=1; break; } - [ -f "$D/STOP" ] && break - if pgrep -f "[a]pi_server" >/dev/null; then dead=0; else dead=$((dead + 1)); fi - if [ "$dead" -ge 3 ]; then log "server process gone during boot (after $((i*10))s)"; break; fi -done -COLD=$(( $(date +%s) - T0 )) -if [ "$ready" != "1" ]; then - log "NOT READY after ${COLD}s" - tail -40 "$D/serve.log" | tee -a "$D/driver.log" - sel_snap notready - teardown; hygiene - echo "NOT_READY" >"$D/status.txt" - exit 1 -fi -log "READY t=${COLD}s" -echo "$COLD" >"$D/cold_compile_seconds.txt" -sel_snap after_boot || { teardown; hygiene; exit 1; } - -# --- Warmup (throwaway: cold-start pollution is real on this stack) --------- -if [ "$DO_WARMUP" = "1" ]; then - for wl in "1024 64 999" "16384 64 998"; do - set -- $wl - log "warmup in=$1 out=$2" - ( cd "$D" && "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model "$MODEL" --served-model-name flash-next --dataset-name random \ - --random-input-len "$1" --random-output-len "$2" --num-prompts 1 --max-concurrency 1 \ - --ignore-eos --request-rate inf --seed "$3" --temperature 0 \ - --save-result --result-dir "$D/warmup_$1" ) >"$D/warmup_$1.log" 2>&1 - done - log "warmup done" - sel_snap after_warmup || { teardown; hygiene; exit 1; } -fi - -# --- Coherence (documented probe) ------------------------------------------ -log "coherence probe start" -"$V/bin/python" "$T/tools/rdna2_028/probe_w4a8.py" "http://127.0.0.1:$PORT/v1" flash-next 1 >"$D/coherence.txt" 2>&1 -log "coherence probe done rc=$?" -grep -E "^\[|OK /" "$D/coherence.txt" | tee -a "$D/driver.log" - -# --- Measured cells -------------------------------------------------------- -IFS='|' read -ra _cells <<<"$CELLS" -for cell in "${_cells[@]}"; do - # CELLS field order matches the established protocol: "n_prompts in_len out_len seed". - set -- $cell - N=$1; IN=$2; OUTL=$3; SEED=$4 - if [ "$N" -gt 8 ] || [ "$IN" -lt 64 ] || [ "$OUTL" -lt 1 ]; then - log "ABORT bad cell spec: N=$N IN=$IN OUT=$OUTL (want N<=8, IN>=64, OUT>=1)" - echo "BAD_CELL_SPEC" >"$D/status.txt" - teardown; hygiene - exit 2 - fi - CD="$D/cells/c${IN}_${N}" - log "bench c=$N in=$IN out=$OUTL start" - ( cd "$D" && "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model "$MODEL" --served-model-name flash-next --dataset-name random \ - --random-input-len "$IN" --random-output-len "$OUTL" --num-prompts "$N" --max-concurrency "$N" \ - --ignore-eos --request-rate inf --seed "$SEED" --temperature 0 \ - --save-result --result-dir "$CD" ) >"$CD.log" 2>&1 - log "bench c=$N in=$IN done: $(grep -m1 'Output token throughput' "$CD.log" | tr -s ' ')" - # Per-cell coherence (France -> Paris, 2+2 -> 4) + garbage guard text captured raw. - "$V/bin/python" - "$PORT" "$CD" <<'PY' >"$CD/coherence.txt" 2>&1 -import json, sys, urllib.request -port, out = sys.argv[1], sys.argv[2] -def ask(prompt, mt=16): - body = json.dumps({"model": "flash-next", "prompt": prompt, "max_tokens": mt, - "temperature": 0.0, "ignore_eos": True}).encode() - req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions", data=body, - headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(req, timeout=600) as r: - return json.load(r)["choices"][0]["text"] or "" -for name, prompt, expect in [("france", "The capital of France is", "paris"), - ("math", "2 + 2 =", "4")]: - try: - t = ask(prompt) - except Exception as e: # noqa: BLE001 - t = f"" - ok = expect in t.lower() - print(f"[{name}] coherent={ok} | {t[:120]!r}") -PY - sel_snap "after_c${N}_${IN}" || { teardown; hygiene; exit 1; } -done - -# --- Markers --------------------------------------------------------------- -{ - echo "=== 'W4A8 sdot4 path active' (expected iff W4A8=1 AND a dense quantized linear ran) ===" - grep -m4 "W4A8 sdot4 path active" "$D/serve.log" || echo "(absent)" - echo "=== 'W4A8-DEBUG' fast-path shapes ===" - grep -h "W4A8-DEBUG" "$D/serve.log" | sort -u | head -30 || echo "(none)" - echo "=== rdna_ar lines (expected: one-shot active on every rank) ===" - grep -h "rdna_ar:" "$D/serve.log" | sort -u || echo "(none)" - echo "=== all-reduce backend selection ===" - grep -m2 "all-reduce backends" "$D/serve.log" || echo "(none)" - echo "=== attention backend ===" - grep -m2 "attention backend\|Using RDNA_ATTN\|Attention backend" "$D/serve.log" || echo "(none)" - echo "=== shm_broadcast stalls ===" - grep -c "no available shared memory block" "$D/serve.log" || true -} >"$D/markers.txt" 2>&1 -sed -n '1,40p' "$D/markers.txt" | tee -a "$D/driver.log" - -{ - echo "=== SpecDecoding metrics (serve log) ===" - grep "SpecDecoding metrics" "$D/serve.log" || echo "(none)" -} >"$D/acceptance.txt" 2>&1 - -# --- Extract (CSV) --------------------------------------------------------- -"$V/bin/python" "$T/tools/rdna2_028/flashnext_w4a8_extract.py" "$D" | tee "$D/cells.csv" | tail -8 - -teardown -hygiene -sel_snap after_teardown || true -log "done $TAG (W4A8=$W4A8 MTP=$MTP)" -echo "OK" >"$D/status.txt" diff --git a/tools/rdna2_028/flashnext_w4a8_extract.py b/tools/rdna2_028/flashnext_w4a8_extract.py deleted file mode 100644 index d2ea6a30b344..000000000000 --- a/tools/rdna2_028/flashnext_w4a8_extract.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Flatten the bench-serve JSON result files of a Flash-Next W4A8 arm to CSV. - -Usage: flashnext_w4a8_extract.py - -Finds /cells/c_/openai-*.json (plus warmup dirs), picks the -newest JSON per dir, and prints one CSV row per measured cell. All metrics are -read defensively: missing keys become empty fields, never a crash. -""" -import csv -import glob -import json -import os -import re -import sys - - -def pick_json(d): - cands = sorted(glob.glob(os.path.join(d, "*.json")), key=os.path.getmtime) - return cands[-1] if cands else None - - -def dims_from_log(d): - """The bench JSON omits the random lengths; the sibling .log args carry them.""" - p = d + ".log" - try: - with open(p) as f: - head = f.read() - except OSError: - return "", "" - m = re.search(r"random_input_len=(\d+),\s*random_output_len=(\d+)", head) - return (m.group(1), m.group(2)) if m else ("", "") - - -def num(d, *keys): - for k in keys: - if k in d and d[k] is not None: - return d[k] - return "" - - -def main() -> None: - arm = sys.argv[1] - arm_name = os.path.basename(arm) - w = csv.writer(sys.stdout) - w.writerow([ - "arm", "dir", "n", "in_len", "out_len", "ok", "dur_s", "in_tok", "out_tok", - "ttft_mean_ms", "ttft_med_ms", "ttft_p99_ms", - "tpot_mean_ms", "itl_med_ms", "itl_p99_ms", - "out_tok_s_agg", "total_tok_s", "decode_tok_s_per_req", "prefill_tok_s", - "accept_len", "accept_rate", - ]) - dirs = sorted(glob.glob(os.path.join(arm, "cells", "c*_*"))) - dirs += sorted(glob.glob(os.path.join(arm, "warmup_*"))) - for d in dirs: - p = pick_json(d) - if not p: - continue - with open(p) as f: - r = json.load(f) - n = num(r, "num_prompts", "completed") or "" - inlen = num(r, "random_input_len", "max_input_len") or "" - outlen = num(r, "random_output_len", "max_output_len") or "" - if not inlen or not outlen: - ji, jo = dims_from_log(d) - inlen = inlen or ji - outlen = outlen or jo - in_tok = num(r, "total_input_tokens") - out_tok = num(r, "total_output_tokens") - ttft = num(r, "mean_ttft_ms") - tpot = num(r, "mean_tpot_ms") - decode = round(1000.0 / tpot, 2) if isinstance(tpot, (int, float)) and tpot else "" - prefill = "" - if isinstance(ttft, (int, float)) and ttft and isinstance(in_tok, (int, float)): - prefill = round(in_tok / (ttft / 1000.0), 1) - w.writerow([ - arm_name, os.path.basename(d), n, inlen, outlen, - num(r, "completed", "successful_requests"), num(r, "duration"), - in_tok, out_tok, - ttft, num(r, "median_ttft_ms"), num(r, "p99_ttft_ms"), - tpot, num(r, "median_itl_ms"), num(r, "p99_itl_ms"), - num(r, "output_throughput", "output_token_throughput"), - num(r, "total_token_throughput"), - decode, prefill, - num(r, "acceptance_length"), num(r, "acceptance_rate"), - ]) - - -if __name__ == "__main__": - main() diff --git a/tools/rdna2_028/flashnext_w4a8_matrix.sh b/tools/rdna2_028/flashnext_w4a8_matrix.sh deleted file mode 100644 index 089421f96984..000000000000 --- a/tools/rdna2_028/flashnext_w4a8_matrix.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# Qwen3.8-Flash-Next W4A8/W4A16 matrix orchestrator (TP=4, FA-RDNA2, RDNA AR=1, -# FULL_AND_PIECEWISE, prefix caching). Four sequential one-engine boots: -# -# 1) W4A8=1 MTP=0 2) W4A8=1 MTP=2 (primary: the HIP W4A8 arm) -# 3) W4A8=0 MTP=0 4) W4A8=0 MTP=2 (same-session W4A16 control) -# -# One global PCI-SERR baseline; a NEW 'PCI SERR' after any phase writes STOP and -# aborts the remaining arms. Each arm is a full tools/rdna2_028/flashnext_w4a8_arm.sh -# run (own tagged cache, warmth, 4 cells). Logs under /home/chenco_adm/w4a8_runs/. -# -# Usage: setsid nohup bash flashnext_w4a8_matrix.sh > /home/chenco_adm/w4a8_runs/matrix.log 2>&1 & -set -uo pipefail - -T=/home/chenco_adm/vllm-rdna-0.28.0 -OUT=${OUT:-/home/chenco_adm/w4a8_runs} -STAMP=${STAMP:-2026-09-29} -RUN=$OUT/${STAMP}_matrix -mkdir -p "$RUN" - -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$RUN/matrix.log"; } -sel_count() { sudo -n ipmitool sel list 2>/dev/null | grep -c 'PCI SERR'; } - -log "=== Flash-Next W4A8 matrix start host=$(hostname) uptime=$(uptime -p) ===" -SERR0=$(sel_count) -log "global PCI-SERR baseline=$SERR0" -echo "$SERR0" >"$RUN/serr_baseline.txt" - -run_arm() { - local arm=$1 w4a8=$2 mtp=$3 port=$4 - local tag="${STAMP}_fn-${arm}-m${mtp}" - log "--- ARM $tag (W4A8=$w4a8 MTP=$mtp port=$port) start ---" - env ARM="$arm" W4A8="$w4a8" MTP="$mtp" PORT="$port" TAG="$tag" OUT="$OUT" \ - SERR_BASE="$SERR0" WARM_TRITON=1 DO_WARMUP=1 \ - bash "$T/tools/rdna2_028/flashnext_w4a8_arm.sh" >>"$RUN/$tag.driver" 2>&1 - local rc=$? - local c; c=$(sel_count) - log "--- ARM $tag end rc=$rc PCI-SERR=$c ---" - if [ "$c" -gt "$SERR0" ]; then - echo "STOP new PCI SERR in $tag ($c > $SERR0)" >"$RUN/STOP" - log "STOP: new PCI SERR in $tag; aborting remaining arms" - return 1 - fi - [ "$rc" -ne 0 ] && { log "arm $tag rc=$rc (see $RUN/$tag.driver)"; } - return 0 -} - -run_arm w4a8on 1 0 18260 || exit 1 -run_arm w4a8on 1 2 18261 || exit 1 -run_arm w4a8off 0 0 18262 || exit 1 -run_arm w4a8off 0 2 18263 || exit 1 - -log "all arms done; final PCI-SERR=$(( $(sel_count) - SERR0 )) new" -echo "MATRIX_DONE" >"$RUN/matrix.status" -log "=== Flash-Next W4A8 matrix end ===" diff --git a/tools/rdna2_028/fp32_accum_matrix.sh b/tools/rdna2_028/fp32_accum_matrix.sh deleted file mode 100644 index 9314743ad938..000000000000 --- a/tools/rdna2_028/fp32_accum_matrix.sh +++ /dev/null @@ -1,287 +0,0 @@ -#!/usr/bin/env bash -# fp32-accum matrix driver — Flash-Next, TP=4, FULL_AND_PIECEWISE cudagraphs, -# prefix caching, FA-RDNA2, frozen TunableOp rows. Runs on GPUs 4-7. -# -# Arms (per-arm cells): -# fp32-m0 / fp32-m2 : VLLM_RDNA2_MOE_FP32_ACCUM=1 (default), W4A8=0, 4 cells -# cas-m0 / cas-m2 : VLLM_RDNA2_MOE_FP32_ACCUM=0 (legacy CAS), W4A8=0, c=1 cells -# w4a8-m0 : VLLM_RDNA2_MOE_FP32_ACCUM=1 + VLLM_RDNA2_W4A8_SDOT4=1, 1 cell -# -# Env: ARMS (space list), OUT, MODEL, TAG_PREFIX, HIP_DEVICES. No /tmp. -# Logs + captures live under /home/chenco_adm/w4a8_runs/. -set -uo pipefail - -V=/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0 -T=/home/chenco_adm/vllm-rdna-0.28.0 -MODEL=${MODEL:-/home/chenco_adm/hfcache/hub/models--wtdcode--Qwen3.8-Flash-Next-AWQ-W4A16/snapshots/0939125b929543a783ce700c90e36dd1a575c00c} -OUT=${OUT:-/home/chenco_adm/w4a8_runs} -TAG_PREFIX=${TAG_PREFIX:-2026-09-30_fp32-accum-matrix} -HIP_DEVICES=${HIP_DEVICES:-4,5,6,7} -ARMS=${ARMS:-"fp32-m0 fp32-m2 cas-m0 cas-m2 w4a8-m0"} -TRITON_TEMPLATE=${TRITON_TEMPLATE:-$T/cache/triton} - -ROCM_SDK_LIB=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -ROCM_SDK=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/rocm_sysdeps/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - -DRIVER_LOG=$OUT/fp32_accum_matrix.log -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$DRIVER_LOG"; } - -# The api_server -> EngineCore spawn drops launcher-passed env vars, so the -# per-arm fork knobs (VLLM_RDNA2_MOE_FP32_ACCUM / VLLM_RDNA2_W4A8_SDOT4) never -# reach the workers via the launch env. This installs (idempotently) a venv -# sitecustomize hook that re-loads the per-arm mode file into `os.environ` in -# every process, which the worker's _custom_ops import then reads. -ensure_fork_mode_hook() { - local sc=$V/lib/python3.12/site-packages/sitecustomize.py - grep -q "rdna_fork_mode" "$sc" 2>/dev/null && return 0 - python3 - "$sc" <<'HOOKPY' -import sys -p = sys.argv[1] -block = ( - "_mode_file = os.path.expanduser(\"~/.cache/rdna_fork_mode\")\n" - "if os.path.exists(_mode_file):\n" - " try:\n" - " with open(_mode_file) as _f:\n" - " for _line in _f:\n" - " _line = _line.strip()\n" - " if \"=\" in _line:\n" - " _k, _v = _line.split(\"=\", 1)\n" - " os.environ[_k.strip()] = _v.strip()\n" - " except Exception:\n" - " pass\n" -) -with open(p) as f: - src = f.read() -out, done = [], False -for ln in src.split("\n"): - out.append(ln) - if not done and ln == "import os": - out.append(block.rstrip("\n")) - done = True -with open(p, "w") as f: - f.write("\n".join(out)) -HOOKPY -} - -# Robust "ours" detection: catches vllm processes whose venv python is a -# symlink to /usr/bin/python3.12 (so a resolved-exe path match would miss -# them). Matches on the distinctive VLLM::*/PleOffloadWorker comm OR the venv -# path in the cmdline (api_server / resource_tracker / bench driver). -our_pids() { - local p - for p in $(pgrep -f "VLLM::Worker|VLLM::EngineCore|entrypoints.openai.api_server|entrypoints.cli.main serve|PleOffloadWorker|resource_tracker" 2>/dev/null); do - if grep -qE "^VLLM::|^PleOffloadWorker" "/proc/$p/comm" 2>/dev/null; then - echo "$p" - elif tr '\0' '\n' < "/proc/$p/cmdline" 2>/dev/null | grep -q "venv-7.14.0_0.28.0"; then - echo "$p" - fi - done -} - -hygiene() { - local n=0 left p - for p in $(our_pids); do kill -TERM "$p" 2>/dev/null && n=$((n + 1)); done - [ "$n" -gt 0 ] && sleep 8 - for p in $(our_pids); do kill -KILL "$p" 2>/dev/null; done - sleep 2 - left=$(our_pids | tr '\n' ' ') - log "hygiene: terminated=$n remaining_our=[${left% }]" - [ -n "${left// /}" ] && return 1 - return 0 -} - -sel_count() { timeout 10 sudo -n ipmitool sel list 2>/dev/null | grep -c 'PCI SERR'; } -sel_snap() { - local tag=$1 c - c=$(sel_count) - log "SEL[$tag] PCI-SERR=$c base=${SERR0:-?} uptime=$(uptime -p)" - if [ -n "${SERR0:-}" ] && [ "${c:-0}" -gt "$SERR0" ]; then - echo "STOP NEW PCI SERR after $tag ($c > $SERR0)" >"$OUT/STOP" - log "STOP: new PCI SERR after $tag"; return 1 - fi - return 0 -} - -teardown() { - local api p - api=$(grep -oE "APIServer pid=[0-9]+" "$D/serve.log" 2>/dev/null | head -1 | cut -d= -f2) - [ -n "$api" ] && { kill -TERM "$api" 2>/dev/null; sleep 10; } - for p in $(our_pids); do kill -TERM "$p" 2>/dev/null; done - sleep 6 - for p in $(our_pids); do kill -KILL "$p" 2>/dev/null; done - sleep 2 -} - -# Per-arm cell list. -cells_for_arm() { - case "$1" in - fp32-m0|fp32-m2) echo "1 16384 1024 111|8 16384 1024 112|1 1024 512 221|8 1024 512 222" ;; - cas-m0|cas-m2) echo "1 16384 1024 111|1 1024 512 221" ;; - w4a8-m0) echo "8 16384 1024 112" ;; - *) echo "" ;; - esac -} - -run_arm() { - local arm=$1 FP32 W4A8 MTP ATTN PORT - case "$arm" in - fp32-m0) FP32=1 W4A8=0 MTP=0 ATTN=fa PORT=18410 ;; - fp32-m2) FP32=1 W4A8=0 MTP=2 ATTN=fa PORT=18411 ;; - cas-m0) FP32=0 W4A8=0 MTP=0 ATTN=fa PORT=18412 ;; - cas-m2) FP32=0 W4A8=0 MTP=2 ATTN=fa PORT=18413 ;; - w4a8-m0) FP32=1 W4A8=1 MTP=0 ATTN=fa PORT=18414 ;; - *) log "unknown arm: $arm"; return 1 ;; - esac - local CELLS - CELLS=$(cells_for_arm "$arm") - local TAG=${TAG_OVERRIDE:-${TAG_PREFIX}_$arm} - local CACHE=$T/cache/arm-fp32-$arm - D=$OUT/$TAG - mkdir -p "$D/cells" - - log "=== ARM=$arm FP32=$FP32 W4A8=$W4A8 MTP=$MTP ATTN=$ATTN PORT=$PORT HIP=$HIP_DEVICES TAG=$TAG ===" - hygiene || { log "aborting: hygiene failed"; return 1; } - if ss -ltn 2>/dev/null | grep -q ":$PORT "; then log "aborting: port $PORT bound"; return 1; fi - rm -rf "$CACHE"; mkdir -p "$CACHE/inductor" "$CACHE/extensions" - if [ -d "$TRITON_TEMPLATE" ]; then - cp -a "$TRITON_TEMPLATE" "$CACHE/triton" - log "triton cache seeded ($(ls "$CACHE/triton" | wc -l) entries)" - else mkdir -p "$CACHE/triton"; fi - rm -f "$CACHE/rdna_ar_wedged" - SERR0=$(sel_count) - log "arm cache=$CACHE baseline PCI-SERR=$SERR0" - sel_snap preflight || return 2 - - ROCK_MODE=$HOME/.cache/rdna_fork_mode - mkdir -p "$HOME/.cache" - printf 'VLLM_RDNA2_MOE_FP32_ACCUM=%s\n' "$FP32" > "$ROCK_MODE" - [ "$W4A8" = "1" ] && printf 'VLLM_RDNA2_W4A8_SDOT4=1\n' >> "$ROCK_MODE" - - T0=$(date +%s) - setsid nohup env \ - VENV="$V" VLLM_TREE="$T" MTP="$MTP" ATTN="$ATTN" TP=4 PORT="$PORT" MODEL="$MODEL" \ - VLLM_CACHE_ROOT="$CACHE" TORCHINDUCTOR_CACHE_DIR="$CACHE/inductor" \ - TRITON_CACHE_DIR="$CACHE/triton" TORCH_EXTENSIONS_DIR="$CACHE/extensions" \ - HIP_VISIBLE_DEVICES="$HIP_DEVICES" \ - VLLM_RDNA2_MOE_FP32_ACCUM="$FP32" \ - VLLM_RDNA2_W4A8_SDOT4="$W4A8" \ - VLLM_RDNA_AR=1 VLLM_RDNA_AR_MAX_KB=64 VLLM_RDNA_AR_ONESHOT_KB=64 \ - VLLM_RDNA_AR_BLOCKS=0 VLLM_RDNA_AR_PACE=0 VLLM_FORCE_CUSTOM_ALL_REDUCE=0 \ - NCCL_P2P_LEVEL=pxb RCCL_P2P_NET_DISABLE=1 RCCL_P2P_BATCH_ENABLE=1 \ - NCCL_PROTO=Simple RCCL_MSCCL_ENABLE=0 PYTHONFAULTHANDLER=1 \ - bash "$T/tools/rdna/serve_gfx1030_flashnext_mtp.sh" >"$D/serve.log" 2>&1 /dev/null | grep -q flash-next && { ready=1; break; } - [ -f "$OUT/STOP" ] && break - if pgrep -f "[a]pi_server" >/dev/null; then dead=0; else dead=$((dead + 1)); fi - if [ "$dead" -ge 3 ]; then log "server gone during boot (after $((i*10))s)"; break; fi - done - local COLD=$(( $(date +%s) - T0 )) - if [ "$ready" != "1" ]; then - log "NOT READY after ${COLD}s"; tail -30 "$D/serve.log" | tee -a "$DRIVER_LOG" | tail -20 - teardown; hygiene; echo "NOT_READY" >"$D/status.txt"; return 1 - fi - log "READY t=${COLD}s"; echo "$COLD" >"$D/cold_compile_seconds.txt" - sel_snap after_boot || { teardown; hygiene; return 1; } - - # Warmup (throwaway) then coherence. - for wl in "1024 64 999" "16384 64 998"; do - set -- $wl - ( cd "$D" && "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model "$MODEL" --served-model-name flash-next --dataset-name random \ - --random-input-len "$1" --random-output-len "$2" --num-prompts 1 --max-concurrency 1 \ - --ignore-eos --request-rate inf --seed "$3" --temperature 0 \ - --save-result --result-dir "$D/warmup_$1" ) >"$D/warmup_$1.log" 2>&1 - done - log "warmup done" - "$V/bin/python" "$T/tools/rdna2_028/probe_w4a8.py" "http://127.0.0.1:$PORT/v1" flash-next 1 >"$D/coherence.txt" 2>&1 - log "coherence rc=$? $(grep -c 'OK ' "$D/coherence.txt")/$(grep -cE '^\[' "$D/coherence.txt") OK" - - # Measured cells. - local cell - IFS='|' read -ra _cells <<<"$CELLS" - for cell in "${_cells[@]}"; do - set -- $cell - local N=$1 IN=$2 OUTL=$3 SEED=$4 - local CD="$D/cells/c${IN}_${N}" - log "bench c=$N in=$IN start" - ( cd "$D" && "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model "$MODEL" --served-model-name flash-next --dataset-name random \ - --random-input-len "$IN" --random-output-len "$OUTL" --num-prompts "$N" --max-concurrency "$N" \ - --ignore-eos --request-rate inf --seed "$SEED" --temperature 0 \ - --save-result --result-dir "$CD" ) >"$CD.log" 2>&1 - log "bench c=$N in=$IN done: $(grep -m1 'Output token throughput' "$CD.log" | tr -s ' ')" - "$V/bin/python" - "$PORT" "$CD" <<'PY' >"$CD/coherence.txt" 2>&1 -import json, sys, urllib.request -port, out = sys.argv[1], sys.argv[2] -def ask(p, mt=16): - body = json.dumps({"model": "flash-next", "prompt": p, "max_tokens": mt, - "temperature": 0.0, "ignore_eos": True}).encode() - r = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions", data=body, - headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(r, timeout=600) as resp: - return json.load(resp)["choices"][0]["text"] or "" -for name, p, exp in [("france", "The capital of France is", "paris"), ("math", "2 + 2 =", "4")]: - try: t = ask(p) - except Exception as e: t = f"" - print(f"[{name}] coherent={exp in t.lower()} | {t[:120]!r}") -PY - sel_snap "after_c${N}_${IN}" || { teardown; hygiene; return 1; } - done - - # Markers. - { - echo "=== W4A8 MoE marker (expected iff W4A8=1) ===" - grep -m4 "W4A8\|w4a8\|sdot4" "$D/serve.log" | grep -v "W4A8-CALLTRACE" | head -8 || echo "(none)" - echo "=== fp32 accum ===" - grep -m2 "fp32_accum\|FP32_ACCUM\|fp32 accum" "$D/serve.log" | head -4 || echo "(none)" - echo "=== TunableOp files ===" - grep -h "reading tuning results from\|TunableOp lookup\|record" "$D/serve.log" | sort -u | head -6 || echo "(none)" - echo "=== rdna_ar ===" - grep -h "rdna_ar:" "$D/serve.log" | sort -u | head -4 || echo "(none)" - echo "=== attention backend ===" - grep -h "RDNA_ATTN\|TRITON_ATTN\|attention backend" "$D/serve.log" | sort -u | head -4 || echo "(none)" - } >"$D/markers.txt" 2>&1 - tail -14 "$D/markers.txt" | tee -a "$DRIVER_LOG" - - "$V/bin/python" "$T/tools/rdna2_028/flashnext_w4a8_extract.py" "$D" >"$D/cells.csv" 2>"$D/extract.err" || true - cat "$D/cells.csv" | tee -a "$DRIVER_LOG" | tail -8 - grep "SpecDecoding metrics" "$D/serve.log" >"$D/acceptance.txt" 2>/dev/null || true - - teardown - hygiene - sel_snap after_teardown || true - rm -f "$ROCK_MODE" - log "done $arm (FP32=$FP32 W4A8=$W4A8 MTP=$MTP ATTN=$ATTN)" - echo "OK" >"$D/status.txt" -} - -# --- Driver ---------------------------------------------------------------- -ensure_fork_mode_hook -SERR0=$(sel_count) -log "=== fp32_accum_matrix start ARMS=[$ARMS] HIP=$HIP_DEVICES PCI-SERR=$SERR0 uptime=$(uptime -p) ===" -hygiene || { log "hygiene failed at start"; exit 1; } - -rc=0 -for arm in $ARMS; do - log "---------------------------------------------------------------" - if ! run_arm "$arm"; then - log "ARM FAIL: $arm" - rc=1 - grep -q "PCI SERR" "$OUT/STOP" 2>/dev/null && { log "stopping: PCI SERR"; break; } - hygiene || { log "hygiene failed after $arm"; break; } - fi -done - -SERR1=$(sel_count) -log "=== fp32_accum_matrix done rc=$rc PCI-SERR=$SERR1 (base $SERR0) ===" -[ "$SERR1" -gt "$SERR0" ] && { echo "STOP NEW PCI SERR" >"$OUT/STOP"; log "STOP: new PCI SERR"; exit 2; } -echo "DONE rc=$rc" >"$OUT/fp32_accum_matrix.status" -exit $rc diff --git a/tools/rdna2_028/fp_aroff_27b.sh b/tools/rdna2_028/fp_aroff_27b.sh deleted file mode 100644 index 436389dc0c65..000000000000 --- a/tools/rdna2_028/fp_aroff_27b.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# 27B dense W4A16 bring-up at TP=4 on the FULL HIP path with the fork's -# PCIe-peer all-reduce DISABLED (RDNA_AR=0) and FULL_AND_PIECEWISE cudagraphs. -# Fresh arm-tagged caches force a clean W4A16 compile (the shared cache holds a -# contaminated W4A8 graph that replays the `duct` collapse). -# -# IPMI is sampled before, after each cell, and after teardown; a new PCI SERR -# aborts the remaining cells and drops a STOP marker. -# -# Usage: TAG=2026-09-29_tp4-fp-aroff bash fp_aroff_27b.sh -set -uo pipefail -V=/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0 -T=/home/chenco_adm/vllm-rdna-0.28.0 -CACHE=$T/cache/hip-fp-aroff -MODEL=${MODEL:-/home/chenco_adm/.cache/huggingface/hub/models--cyankiwi--Qwen3.8-27B-AWQ-INT4/snapshots/63768c10df38c0395e12ef49edac1bd539eaeeea} -TAG=${TAG:-2026-09-29_tp4-fp-aroff} -PORT=${PORT:-18240} -OUT=${OUT:-/home/chenco_adm/w4a8_runs} -D=$OUT/$TAG -rm -rf "$D"; mkdir -p "$D" -ROCK_SDK_LIB=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -ROCK_SDK=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$ROCK_SDK_LIB:$ROCK_SDK/host-math/lib:$ROCK_SDK/rocm_sysdeps/lib:$ROCK_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib" - -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$D/driver.log"; } - -sel_snapshot() { - local tag=$1 - sudo -n ipmitool sel list 2>/dev/null > "$D/$tag.sel.txt" - local serr crit ecc - serr=$(grep -c 'PCI SERR' "$D/$tag.sel.txt") - crit=$(grep -c 'Critical Interrupt' "$D/$tag.sel.txt") - ecc=$(grep -c 'Correctable ECC' "$D/$tag.sel.txt") - { - echo "=== [$tag] $(date -Is) uptime: $(uptime -p)" - echo "PCI SERR: $serr (baseline $SERR0) Critical Interrupt: $crit Correctable ECC: $ecc" - tail -3 "$D/$tag.sel.txt" - } | tee -a "$D/ipmi.log" - if [ "$serr" -gt "$SERR0" ]; then - echo "STOP: NEW PCI SERR after $tag" | tee -a "$D/ipmi.log" - echo "STOP NEW PCI SERR after $tag" > "$D/STOP" - return 1 - fi - return 0 -} - -teardown() { - local api - api=$(grep -oE "APIServer pid=[0-9]+" "$D/serve.log" 2>/dev/null | head -1 | cut -d= -f2) - [ -n "$api" ] && kill -TERM "$api" 2>/dev/null && sleep 10 - for p in $(pgrep -f "VLLM::Worker|VLLM::EngineCore|entrypoints.cli.main serve" 2>/dev/null); do - tr '\0' '\n' < "/proc/$p/environ" 2>/dev/null | grep -q "VLLM_CACHE_ROOT=$CACHE" && kill -9 "$p" 2>/dev/null - done -} - -# --- Preflight: baseline fabric events, clean caches, free port --- -log "preflight: TAG=$TAG PORT=$PORT CACHE=$CACHE" -mkdir -p "$(dirname "$CACHE")" -rm -rf "$CACHE" -mkdir -p "$CACHE/inductor" "$CACHE/triton" "$CACHE/extensions" -SERR0=$(sudo -n ipmitool sel list 2>/dev/null | grep -c 'PCI SERR') -sel_snapshot before || { log "baseline snapshot failed; aborting"; exit 1; } -log "baseline PCI SERR count=$SERR0" - -# --- Launch: FULL HIP path, no fork AR, F&P graphs, fresh caches --- -T0=$(date +%s) -setsid nohup env MTP=0 W4A8=0 RDNA_AR=0 EAGER=0 ATTN=fa TP=4 PORT=$PORT \ - KV=8000000000 SEQS=8 MAXBAT=2048 CG_MODE=FULL_AND_PIECEWISE MODEL="$MODEL" \ - VLLM_CACHE_ROOT="$CACHE" TORCHINDUCTOR_CACHE_DIR="$CACHE/inductor" \ - TRITON_CACHE_DIR="$CACHE/triton" TORCH_EXTENSIONS_DIR="$CACHE/extensions" \ - bash "$T/tools/rdna/serve_gfx1030_27b_dense.sh" > "$D/serve.log" 2>&1 < /dev/null & -log "serve launched pid $! (MTP=0 W4A8=0 RDNA_AR=0 CG_MODE=FULL_AND_PIECEWISE)" - -ready=0 -for i in $(seq 1 360); do - sleep 10 - curl -s --max-time 4 "http://127.0.0.1:$PORT/v1/models" 2>/dev/null | grep -q q27d && { ready=1; break; } -done -COLD=$(( $(date +%s) - T0 )) -if [ "$ready" = "1" ]; then - log "READY t=${COLD}s (cold compile+capture window)" - echo "$COLD" > "$D/cold_compile_seconds.txt" -else - log "NOT READY after ${COLD}s" - tail -40 "$D/serve.log" - sel_snapshot notready - exit 1 -fi -sel_snapshot after_boot || { teardown; exit 1; } - -# --- Warm-up cell (throwaway; cold-start pollution skews the first measured cell) --- -log "warmup cell start" -cd "$D" -"$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model "$MODEL" --served-model-name q27d --dataset-name random \ - --random-input-len 16384 --random-output-len 64 --num-prompts 1 --max-concurrency 1 \ - --ignore-eos --request-rate inf --seed 999 --temperature 0 \ - --save-result --result-dir "$D/warmup" > "$D/warmup.log" 2>&1 -log "warmup cell done" -sel_snapshot after_warmup || { teardown; exit 1; } - -# --- Coherence (greedy) --- -log "coherence probe start" -"$V/bin/python" "$T/tools/rdna2_028/probe_w4a8.py" "http://127.0.0.1:$PORT/v1" q27d 1 | tee "$D/coherence.txt" -log "coherence probe done" - -# --- Marker audit: what should be OFF must not appear --- -{ - echo "=== 'W4A8 sdot4 path active' (must be ABSENT) ===" - grep -n "W4A8 sdot4 path active" "$D/serve.log" || echo "(absent: good)" - echo "=== 'W4A8-DEBUG' (must be ABSENT; debug not set) ===" - grep -n "W4A8-DEBUG" "$D/serve.log" || echo "(absent: good)" - echo "=== 'rdna_ar:' lines ===" - grep -n "rdna_ar:" "$D/serve.log" || echo "(no rdna_ar lines: VLLM_RDNA_AR=0 never constructs the backend)" - echo "=== all-reduce backend selection ===" - grep -n "all-reduce backends" "$D/serve.log" || echo "(none)" - echo "=== 'RDNA_ONESHOT' (must be ABSENT) ===" - grep -n "RDNA_ONESHOT" "$D/serve.log" || echo "(absent: good)" - echo "=== 'Custom allreduce force-enabled' (must be ABSENT) ===" - grep -n "Custom allreduce force-enabled" "$D/serve.log" || echo "(absent: good)" - echo "=== attention backend markers ===" - grep -nE "RDNA_ATTN|RDNA_FA|Using .*attention backend" "$D/serve.log" | head -5 || true -} | tee "$D/markers.txt" - -# --- Measured cells: 16k/1k at c=1, then c=8 --- -for cell in "1 16384 1024 111" "8 16384 1024 112"; do - set -- $cell - log "bench c=$1 in=$2 out=$3 start" - "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model "$MODEL" --served-model-name q27d --dataset-name random \ - --random-input-len "$2" --random-output-len "$3" --num-prompts "$1" --max-concurrency "$1" \ - --ignore-eos --request-rate inf --seed "$4" --temperature 0 \ - --save-result --result-dir "$D/c$2_$1" > "$D/c$2_$1.log" 2>&1 - log "bench c=$1 in=$2 out=$3 done: $(grep -m1 'Output token throughput' "$D/c$2_$1.log" | tr -s ' ')" - sel_snapshot "after_c${1}_${2}" || { teardown; exit 1; } -done - -# --- Teardown + post IPMI --- -teardown -sleep 3 -sel_snapshot after_teardown || true -log "done $TAG" -echo "OK" > "$D/status.txt" diff --git a/tools/rdna2_028/m27b_matrix.sh b/tools/rdna2_028/m27b_matrix.sh deleted file mode 100755 index 45656e63f80a..000000000000 --- a/tools/rdna2_028/m27b_matrix.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# 27B dense matrix: FA-RDNA2 + W4A8 sdot4 + RDNA2 HIP, prefix caching, F&P -# graphs, no --max-model-len. Runs the four production cells and greps the W4A8 -# gate line. Uses the dedicated launcher (tools/rdna/serve_gfx1030_27b_dense.sh). -# Usage: MTP=0 TAG=m27b_m0 bash m27b_matrix.sh -set -uo pipefail -V=/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0 -T=/home/chenco_adm/vllm-rdna-0.28.0 -MODEL=${MODEL:-/home/chenco_adm/.cache/huggingface/hub/models--cyankiwi--Qwen3.8-27B-AWQ-INT4/snapshots/63768c10df38c0395e12ef49edac1bd539eaeeea} -MTP=${MTP:-0} -TAG=${TAG:-m27b_mtp$MTP} -PORT=${PORT:-18220} -OUT=${OUT:-/home/chenco_adm/w4a8_runs} -D=$OUT/$TAG -mkdir -p "$D" -ROCM_SDK_LIB=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -ROCM_SDK=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/rocm_sysdeps/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib" - -setsid nohup env MTP=$MTP W4A8=${W4A8:-0} ATTN=fa TP=${TP:-4} PORT=$PORT \ - KV=${KV:-8000000000} SEQS=${SEQS:-8} MAXBAT=${MAXBAT:-2048} \ - CG_MODE=${CG_MODE:-FULL_AND_PIECEWISE} MODEL="$MODEL" \ - bash "$T/tools/rdna/serve_gfx1030_27b_dense.sh" > "$D/serve.log" 2>&1 < /dev/null & -echo "serve pid $! (MTP=$MTP)" -ready=0 -for i in $(seq 1 72); do - sleep 15 - curl -s --max-time 4 "http://127.0.0.1:$PORT/v1/models" 2>/dev/null | grep -q q27d && { ready=1; echo "READY t=$((i*15))s"; break; } -done -[ "$ready" = "1" ] || { echo "NOT READY"; tail -20 "$D/serve.log"; exit 1; } -for p in "The capital of France is" "2 + 2 ="; do - echo "--- $p" - curl -s --max-time 180 -X POST "http://127.0.0.1:$PORT/v1/completions" -H 'Content-Type: application/json' \ - -d "{\"model\":\"q27d\",\"prompt\":\"$p\",\"max_tokens\":12,\"temperature\":0}" \ - | python3 -c "import sys,json;d=json.load(sys.stdin);print(repr(d['choices'][0]['text']))" 2>/dev/null || echo "(no response)" -done | tee "$D/coherence.txt" -cd "$D" -for cell in "1 16384 1024 111" "8 16384 1024 112" "1 1024 512 221" "8 1024 512 222"; do - set -- $cell - echo "--- bench c=$1 in=$2 out=$3" - "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model "$MODEL" --served-model-name q27d --dataset-name random \ - --random-input-len "$2" --random-output-len "$3" --num-prompts "$1" --max-concurrency "$1" \ - --ignore-eos --request-rate inf --seed "$4" --save-result --result-dir "$D/c$2_$1" > "$D/c$2_$1.log" 2>&1 - grep -E "Mean TTFT|Mean TPOT|Mean ITL|Input token throughput|Output token throughput" "$D/c$2_$1.log" | tail -5 -done -echo "--- W4A8 gate:"; grep -m2 "W4A8 sdot4 path active" "$D/serve.log" || echo "(not taken)" -api=$(grep -oE "APIServer pid=[0-9]+" "$D/serve.log" | head -1 | cut -d= -f2) -[ -n "$api" ] && kill -TERM "$api" 2>/dev/null && sleep 10 -for p in $(pgrep -f "VLLM::Worker|VLLM::EngineCore|entrypoints.cli.main serve" 2>/dev/null); do tr '\0' '\n' < /proc/$p/environ 2>/dev/null | grep -q "VLLM_CACHE_ROOT=$T" && kill -9 $p 2>/dev/null; done -echo "done $TAG" \ No newline at end of file diff --git a/tools/rdna2_028/parse_kernel_sass.py b/tools/rdna2_028/parse_kernel_sass.py deleted file mode 100644 index 444ae0a39382..000000000000 --- a/tools/rdna2_028/parse_kernel_sass.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -"""Print per-kernel register/spill metadata from clang -S AMDGPU output.""" - -import re -import sys - -txt = open(sys.argv[1]).read().splitlines() -cur = {} -rows = [] - -KEYS = ( - "sgpr_count", - "sgpr_spill_count", - "vgpr_count", - "vgpr_spill_count", - "private_segment_fixed_size", - "group_segment_fixed_size", -) - -for line in txt: - s = line.strip() - m = re.match(r"\.name:\s+(_Z\S+)", s) - if m: - if cur.get("name"): - rows.append(cur) - cur = {"name": m.group(1)} - continue - for key in KEYS: - m = re.match(r"\." + key + r":\s+(\d+)", s) - if m and cur.get("name") and key not in cur: - cur[key] = int(m.group(1)) - -if cur.get("name"): - rows.append(cur) - -for r in rows: - n = r["name"] - n = n.replace("_ZN4vllm14moe_w4a8_rdna220moe_w4a8_gemm_kernelI", "GEMM<") - n = n.replace("_ZN4vllm12explore_w4a816w4a8_gemm_kernelI", "DGMM<") - n = n.replace("_ZN4vllm12explore_w4a821w4a8_act_quant_kernelI", "ACTQ<") - n = n.replace("_Z24moe_gemm_q4_kernel_rdna2I", "W4A16M<") - print( - "%-34s sgpr=%4s sp=%3s vgpr=%4s vp=%3s priv=%s" - % ( - n[:34], - r.get("sgpr_count", "?"), - r.get("sgpr_spill_count", "?"), - r.get("vgpr_count", "?"), - r.get("vgpr_spill_count", "?"), - r.get("private_segment_fixed_size", "?"), - ) - ) diff --git a/tools/rdna2_028/pr30_b1_probe.sh b/tools/rdna2_028/pr30_b1_probe.sh deleted file mode 100644 index a9ca7b5daad1..000000000000 --- a/tools/rdna2_028/pr30_b1_probe.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# PR #30 B1: run the compile-dispatch probe on three arms and compare. -# python = default inline dispatch (today) -# op = VLLM_RDNA2_W4A16_RUNTIME_DISPATCH=1 (candidate fix) -# eager = --enforce-eager (no compile; selector follows M) -# -# Defaults are the serving-matrix config (27B AWQ, TP=4, F&P) so every arm -# answers the H1 question on the same model/CG mode the matrix runs at. The -# *graph-ops* evidence is identical at TP=1 and TP=4 because torch.compile -# traces apply_weights once with the trace-time M, so a TP=1 mechanism arm -# does not change the verdict. If you must run TP=1 (e.g. to fit a model on -# 1x GPU), set TP=1 *and* MECHANISM_ONLY=1 — the driver logs the label -# and the record's "mechanism_only" flag is set so H1 cannot be drawn from -# TP=1 perf numbers. -# -# All knobs are env args (no hardcoding beyond the serving-matrix defaults): -# V venv path -# T tree path -# MODEL model snapshot dir -# TP tensor-parallel size (default 4) -# CG_MODE cudagraph mode (default FULL_AND_PIECEWISE) -# MAXLEN max_model_len (default 4096) -# GMEM gpu-memory-utilization (default 0.85) -# TAG results dir suffix (default 2026-09-30_pr30-b1) -# OUT runs root (default /home/chenco_adm/w4a8_runs) -# MECHANISM_ONLY=1 marks this run as mechanism-only (TP=1 is allowed) -# GPUIDS HIP_VISIBLE_DEVICES (default 0,1,2,3) -# -# One engine at a time. Fresh VLLM_CACHE_ROOT per arm. -# bash tools/rdna2_028/pr30_b1_probe.sh -set -uo pipefail -V=${V:-/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0} -T=${T:-/home/chenco_adm/vllm-rdna-0.28.0} -MODEL=${MODEL:-/home/chenco_adm/.cache/huggingface/hub/models--cyankiwi--Qwen3.8-27B-AWQ-INT4/snapshots/63768c10df38c0395e12ef49edac1bd539eaeeea} -TP=${TP:-4} -CG_MODE=${CG_MODE:-FULL_AND_PIECEWISE} -MAXLEN=${MAXLEN:-4096} -GMEM=${GMEM:-0.85} -TAG=${TAG:-2026-09-30_pr30-b1} -OUT=${OUT:-/home/chenco_adm/w4a8_runs} -MECHANISM_ONLY=${MECHANISM_ONLY:-0} -D=$OUT/$TAG -mkdir -p "$D" - -ROCM_SDK_LIB=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -ROCM_SDK=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/rocm_sysdeps/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib" -export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 -export VLLM_WORKER_MULTIPROC_METHOD=spawn GPU_MAX_HW_QUEUES=2 -export VLLM_ROCM_USE_AITER=0 TORCH_BLAS_PREFER_HIPBLASLT=0 -export VLLM_USE_V2_MODEL_RUNNER=1 VLLM_USE_RDNA2_FA=1 -export VLLM_USE_AOT_COMPILE=0 -# The hybrid 27B graph is not saveable on the JIT path (>1 aot_autograd -# artifact -> compiler_interface.is_saveable_2_10 raises). Production 27B -# serves with the compile cache off; computation_graph.py is still written -# to local_cache_dir (backends.py:1359), so H1's graph-op signal survives. -export VLLM_DISABLE_COMPILE_CACHE=1 -export VLLM_RDNA2_W4A8_SDOT4=${W4A8:-0} -export HIP_VISIBLE_DEVICES=${GPUIDS:-0,1,2,3} -source "$T/tools/rdna2_028/tunableop_env.sh" -configure_tunableop "$ROCM_SDK_LIB/librocblas.so.5" "$T/tunableop" 2>&1 | tee -a "$D/driver.log" - -cd "$T" -P="benchmarks.kernels.w4a16_compile_dispatch.probe" -LLMK='{"language_model_only": true, "dtype": "float16"}' -COMPILE="{\"cudagraph_mode\":\"$CG_MODE\",\"compile_ranges_endpoints\":[],\"max_cudagraph_capture_size\":16,\"cudagraph_capture_sizes\":[1,2,4,8,16],\"inductor_compile_config\":{\"combo_kernels\":false}}" -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$D/driver.log"; } - -# Scoped to this venv's exe so a co-tenant (venv-7.14.0) is never touched. -# A killed probe leaves its spawned EngineCore/Workers holding ~27 GiB/GPU; -# without this the next arm boots into "free memory 2.3/29.98 GiB". -our_pids() { - local p exe - for p in $(pgrep -f "entrypoints.cli.main serve|VLLM::Worker|VLLM::EngineCore" 2>/dev/null); do - exe=$(readlink -f "/proc/$p/exe" 2>/dev/null || true) - case "$exe" in */venv-7.14.0_0.28.0/bin/python*) echo "$p" ;; esac - done -} -hygiene() { - local p n=0 - for p in $(our_pids); do kill -TERM "$p" 2>/dev/null && n=$((n + 1)); done - [ "$n" -gt 0 ] && sleep 8 - for p in $(our_pids); do kill -KILL "$p" 2>/dev/null; done - sleep 2 - log "hygiene: terminated=$n remaining=[$(our_pids | tr '\n' ' ')]" -} - -# Hard guard: refuse TP=1 unless explicitly labelled mechanism-only. -if [ "$TP" = "1" ] && [ "$MECHANISM_ONLY" != "1" ]; then - log "FATAL: TP=1 requested without MECHANISM_ONLY=1. Performance numbers at TP=1 do not generalise to the serving-matrix config." - log " Re-run with MECHANISM_ONLY=1 if you only need the graph-ops evidence." - exit 2 -fi - -hygiene -log "model=$MODEL tp=$TP cg=$CG_MODE maxlen=$MAXLEN w4a8=${W4A8:-0} mechanism_only=$MECHANISM_ONLY" -md5sum "$T/vllm/model_executor/kernels/linear/mixed_precision/rdna2_w4a16.py" 2>/dev/null | tee -a "$D/driver.log" - -for arm in python op eager; do - extra=() - [ "$arm" = op ] && extra=(env VLLM_RDNA2_W4A16_RUNTIME_DISPATCH=1) - [ "$arm" = eager ] && extra=(env VLLM_RDNA2_W4A16_RUNTIME_DISPATCH=0) - eager_flag=() - [ "$arm" = eager ] && eager_flag=(--enforce-eager) - log "arm $arm start" - "${extra[@]}" "$V/bin/python" -m $P run \ - --model "$MODEL" --tp "$TP" --max-model-len "$MAXLEN" \ - --gpu-memory-utilization "$GMEM" --llm-kwargs "$LLMK" \ - --compilation-config "$COMPILE" \ - "${eager_flag[@]}" \ - --cache-root "$D/cache-$arm" --json "$D/$arm.json" \ - > "$D/$arm.log" 2>&1 - rc=$? - log "arm $arm done rc=$rc" - if [ "$rc" != "0" ]; then - log "arm $arm FAILED; tail of log:" - tail -30 "$D/$arm.log" | tee -a "$D/driver.log" - fi - hygiene -done - -log "compare" -"$V/bin/python" -m $P compare "$D/python.json" "$D/op.json" "$D/eager.json" 2>&1 | tee -a "$D/driver.log" -log "done" diff --git a/tools/rdna2_028/pr30_b2_rocprof.sh b/tools/rdna2_028/pr30_b2_rocprof.sh deleted file mode 100644 index 881ec160c818..000000000000 --- a/tools/rdna2_028/pr30_b2_rocprof.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# PR #30 B2: rocprofv3 kernel trace per arm, then summarize W4A16 op families. -# Reuses the B1 compile caches so startup is a cache hit (the trace then covers -# the workload, not the compile). The python/op arms replay the same compiled -# graph; only the eager arm differs. One engine at a time. -# -# Defaults are the serving-matrix config (27B AWQ, TP=4, F&P). All knobs are -# env args; no hardcoding beyond serving-matrix defaults: -# V/T/MODEL/TP/CG_MODE/MAXLEN/GMEM/TAG/OUT/B1/GPUIDS — see pr30_b1_probe.sh -# CG_MODE defaults to FULL_AND_PIECEWISE here too. MECHANISM_ONLY=1 lets -# a TP=1 arm run; the resulting kernel counts are still useful for H2. -# -# bash tools/rdna2_028/pr30_b2_rocprof.sh -set -uo pipefail -V=${V:-/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0} -T=${T:-/home/chenco_adm/vllm-rdna-0.28.0} -MODEL=${MODEL:-/home/chenco_adm/.cache/huggingface/hub/models--cyankiwi--Qwen3.8-27B-AWQ-INT4/snapshots/63768c10df38c0395e12ef49edac1bd539eaeeea} -TP=${TP:-4} -CG_MODE=${CG_MODE:-FULL_AND_PIECEWISE} -MAXLEN=${MAXLEN:-4096} -GMEM=${GMEM:-0.85} -B1=${B1:-/home/chenco_adm/w4a8_runs/2026-09-30_pr30-b1} -TAG=${TAG:-2026-09-30_pr30-b2} -OUT=${OUT:-/home/chenco_adm/w4a8_runs} -MECHANISM_ONLY=${MECHANISM_ONLY:-0} -D=$OUT/$TAG -mkdir -p "$D" - -ROCM_SDK_LIB=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -ROCM_SDK=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/rocm_sysdeps/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib" -export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 -export VLLM_WORKER_MULTIPROC_METHOD=spawn GPU_MAX_HW_QUEUES=2 -export VLLM_ROCM_USE_AITER=0 TORCH_BLAS_PREFER_HIPBLASLT=0 -export VLLM_USE_V2_MODEL_RUNNER=1 VLLM_USE_RDNA2_FA=1 VLLM_USE_AOT_COMPILE=0 -export VLLM_DISABLE_COMPILE_CACHE=1 -export VLLM_RDNA2_W4A8_SDOT4=${W4A8:-0} -export HIP_VISIBLE_DEVICES=${GPUIDS:-0,1,2,3} -source "$T/tools/rdna2_028/tunableop_env.sh" -configure_tunableop "$ROCM_SDK_LIB/librocblas.so.5" "$T/tunableop" 2>&1 | tee -a "$D/driver.log" - -cd "$T" -P="benchmarks.kernels.w4a16_compile_dispatch.probe" -LLMK='{"language_model_only": true, "dtype": "float16"}' -COMPILE="{\"cudagraph_mode\":\"$CG_MODE\",\"compile_ranges_endpoints\":[],\"max_cudagraph_capture_size\":16,\"cudagraph_capture_sizes\":[1,2,4,8,16],\"inductor_compile_config\":{\"combo_kernels\":false}}" -ROC=$V/bin/rocprofv3 -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$D/driver.log"; } - -our_pids() { - local p exe - for p in $(pgrep -f "entrypoints.cli.main serve|VLLM::Worker|VLLM::EngineCore" 2>/dev/null); do - exe=$(readlink -f "/proc/$p/exe" 2>/dev/null || true) - case "$exe" in */venv-7.14.0_0.28.0/bin/python*) echo "$p" ;; esac - done -} -hygiene() { - local p n=0 - for p in $(our_pids); do kill -TERM "$p" 2>/dev/null && n=$((n + 1)); done - [ "$n" -gt 0 ] && sleep 8 - for p in $(our_pids); do kill -KILL "$p" 2>/dev/null; done - sleep 2 -} - -if [ "$TP" = "1" ] && [ "$MECHANISM_ONLY" != "1" ]; then - log "FATAL: TP=1 requested without MECHANISM_ONLY=1." - exit 2 -fi -log "model=$MODEL tp=$TP cg=$CG_MODE maxlen=$MAXLEN mechanism_only=$MECHANISM_ONLY" -hygiene - -run_arm() { # $1=name rest=extra env+args after the python cmd - local name=$1; shift - log "arm $name start" - "$ROC" --kernel-trace --output-format csv --output-file "$D/prof_$name" -- \ - "$@" > "$D/${name}_prof.log" 2>&1 - log "arm $name done rc=$?" - hygiene -} - -run_arm python env \ - "$V/bin/python" -m $P run --model "$MODEL" --tp "$TP" --max-model-len "$MAXLEN" \ - --gpu-memory-utilization "$GMEM" --llm-kwargs "$LLMK" --compilation-config "$COMPILE" \ - --cache-root "$B1/cache-python" --repeat 1 --json "$D/python.json" - -run_arm op env VLLM_RDNA2_W4A16_RUNTIME_DISPATCH=1 \ - "$V/bin/python" -m $P run --model "$MODEL" --tp "$TP" --max-model-len "$MAXLEN" \ - --gpu-memory-utilization "$GMEM" --llm-kwargs "$LLMK" --compilation-config "$COMPILE" \ - --cache-root "$B1/cache-op" --repeat 1 --json "$D/op.json" - -run_arm eager env \ - "$V/bin/python" -m $P run --model "$MODEL" --tp "$TP" --max-model-len "$MAXLEN" \ - --gpu-memory-utilization "$GMEM" --llm-kwargs "$LLMK" --enforce-eager \ - --cache-root "$B1/cache-eager" --repeat 1 --json "$D/eager.json" - -log "kernels" -"$V/bin/python" -m $P kernels "$D"/prof_python_kernel_trace.csv \ - "$D"/prof_eager_kernel_trace.csv "$D"/prof_op_kernel_trace.csv 2>&1 | tee -a "$D/driver.log" -log "done" diff --git a/tools/rdna2_028/pr30_b3_driver.sh b/tools/rdna2_028/pr30_b3_driver.sh deleted file mode 100644 index b8933b579eea..000000000000 --- a/tools/rdna2_028/pr30_b3_driver.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# PR #30 B3 + W4A8 freeze check: 27B AWQ dense serve matrix. -# One arm = (RDNA2_W4A16_RUNTIME_DISPATCH, VLLM_RDNA2_W4A8_SDOT4, CG_MODE). -# Arms are run sequentially by this script (one engine at a time). Each arm -# uses a fresh VLLM_CACHE_ROOT (the compile cache key does not include either -# dispatch env var). Coherence + the 4 production cells + duct/P-SERR guards. -# -# ARMS="0 0 1 1" ... see the ARMS list below -# bash tools/rdna2_028/pr30_b3_driver.sh -set -uo pipefail -V=${V:-/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0} -T=${T:-/home/chenco_adm/vllm-rdna-0.28.0} -MODEL=${MODEL:-/home/chenco_adm/.cache/huggingface/hub/models--cyankiwi--Qwen3.8-27B-AWQ-INT4/snapshots/63768c10df38c0395e12ef49edac1bd539eaeeea} -CG_MODE=${CG_MODE:-PIECEWISE} -MTP=${MTP:-0} -OUT=${OUT:-/home/chenco_adm/w4a8_runs} -# "RD W4A8" pairs. -ARMS=${ARMS:-"0 0|1 0|0 1|1 1"} - -ROCM_SDK_LIB=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -ROCM_SDK=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/rocm_sysdeps/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib" -export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 - -serr() { dmesg -T 2>/dev/null | grep -ciE "SERR|AER" || echo 0; } - -# aborts our leaked workers before the next arm boots into "free memory < util"; -# exe-scoped so a co-tenant on another venv is never touched. -our_pids() { - local p exe - for p in $(pgrep -f "entrypoints.cli.main serve|VLLM::Worker|VLLM::EngineCore" 2>/dev/null); do - exe=$(readlink -f "/proc/$p/exe" 2>/dev/null || true) - case "$exe" in */venv-7.14.0_0.28.0/bin/python*) echo "$p" ;; esac - done -} -hygiene() { - local p n=0 - for p in $(our_pids); do kill -TERM "$p" 2>/dev/null && n=$((n + 1)); done - [ "$n" -gt 0 ] && sleep 8 - for p in $(our_pids); do kill -KILL "$p" 2>/dev/null; done - sleep 2 -} - -run_arm() { # $1=rd $2=w4a8 - local rd=$1 w4a8=$2 - local tag="2026-09-30_pr30-b3-rd${rd}-w${w4a8}-${CG_MODE}" - local D=$OUT/$tag port=18320 - rm -rf "$D"; mkdir -p "$D" - local s0; s0=$(serr) - echo "[$(date +%H:%M:%S)] === arm rd=$rd w4a8=$w4a8 cg=$CG_MODE port=$port serr0=$s0 uptime=$(uptime -p) ===" | tee -a "$OUT/pr30_b3_driver.log" - hygiene - - setsid nohup env MTP=$MTP W4A8=$w4a8 ATTN=fa TP=4 PORT=$port \ - KV=${KV:-8000000000} SEQS=${SEQS:-8} MAXBAT=${MAXBAT:-2048} \ - CG_MODE=$CG_MODE MODEL=$MODEL \ - VLLM_CACHE_ROOT="$D/cache" VLLM_DISABLE_COMPILE_CACHE=1 \ - VLLM_RDNA2_W4A16_RUNTIME_DISPATCH=$rd \ - bash "$T/tools/rdna/serve_gfx1030_27b_dense.sh" > "$D/serve.log" 2>&1 /dev/null | grep -q q27d && { ready=1; echo "[$(date +%H:%M:%S)] READY t=$((i*10))s" | tee -a "$D/arm.log"; break; } - done - [ "$ready" = 1 ] || { echo "NOT READY" | tee -a "$D/arm.log"; tail -30 "$D/serve.log"; return 1; } - - # Warmup (throwaway; cold-start pollution is real). - for k in 1 2; do - curl -s --max-time 300 -X POST "http://127.0.0.1:$port/v1/completions" -H 'Content-Type: application/json' \ - -d "{\"model\":\"q27d\",\"prompt\":\"$(printf 'word %.0s' $(seq 1 2000))\",\"max_tokens\":8,\"temperature\":0,\"ignore_eos\":true}" >/dev/null - done - - "$V/bin/python" "$T/tools/rdna2_028/probe_w4a8.py" "http://127.0.0.1:$port/v1" q27d 1 2>&1 | tee -a "$D/arm.log" - - cd "$D" - for cell in "1 16384 1024 111" "8 16384 1024 112" "1 1024 512 221" "8 1024 512 222"; do - set -- $cell - echo "[$(date +%H:%M:%S)] bench c=$1 in=$2 out=$3" | tee -a "$D/arm.log" - "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$port" \ - --model "$MODEL" --served-model-name q27d --dataset-name random \ - --random-input-len "$2" --random-output-len "$3" --num-prompts "$1" --max-concurrency "$1" \ - --ignore-eos --request-rate inf --seed "$4" --temperature 0 \ - --save-result --result-dir "$D/c$2_$1" > "$D/c$2_$1.log" 2>&1 - grep -E "Output token throughput|Mean TTFT|Mean TPOT|Input token throughput" "$D/c$2_$1.log" | tail -4 | tee -a "$D/arm.log" - done - - echo "--- W4A8 gate:" | tee -a "$D/arm.log" - grep -m1 "W4A8 sdot4 path active" "$D/serve.log" || echo "(not taken)" | tee -a "$D/arm.log" - echo "--- duct markers:" | tee -a "$D/arm.log" - grep -c "ductduct" "$D/arm.log" || true - local s1; s1=$(serr) - echo "[$(date +%H:%M:%S)] serr1=$s1 uptime=$(uptime -p)" | tee -a "$D/arm.log" - - local api; api=$(grep -oE "APIServer pid=[0-9]+" "$D/serve.log" | head -1 | cut -d= -f2) - [ -n "$api" ] && kill -TERM "$api" 2>/dev/null && sleep 12 - for p in $(pgrep -f "VLLM::Worker|VLLM::EngineCore|entrypoints.cli.main serve" 2>/dev/null); do - [ -r "/proc/$p/environ" ] || continue - tr '\0' '\n' < "/proc/$p/environ" 2>/dev/null | grep -q "^VLLM_CACHE_ROOT=$D/cache$" && kill -9 "$p" 2>/dev/null - done - sleep 3 - echo "[$(date +%H:%M:%S)] === done rd=$rd w4a8=$w4a8 ===" | tee -a "$OUT/pr30_b3_driver.log" -} - -IFS='|' read -ra _arms <<< "$ARMS" -for a in "${_arms[@]}"; do - set -- $a - run_arm "$1" "$2" -done diff --git a/tools/rdna2_028/probe_force_split.py b/tools/rdna2_028/probe_force_split.py deleted file mode 100644 index de3ded50629e..000000000000 --- a/tools/rdna2_028/probe_force_split.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Time gptq_gemm_rdna2_prefill for one shape with a forced split_k. - -VLLM_RDNA2_PREFILL_FORCE_SPLIT_K is read once per process, so one process per -(m,n,k,force) tuple; the caller loops. Emits one CSV line. Single GPU. -""" - -import argparse -import os -import sys - -import torch - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from probe_w4a8_op import _pack_weights, _quantize_random # noqa: E402 - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--m", type=int, required=True) - ap.add_argument("--k", type=int, required=True) - ap.add_argument("--n", type=int, required=True) - ap.add_argument("--group", type=int, default=32) - ap.add_argument("--force", type=int, required=True) - ap.add_argument("--use-v2", type=int, default=0) - ap.add_argument("--iters", type=int, default=30) - args = ap.parse_args() - - os.environ["VLLM_RDNA2_PREFILL_FORCE_SPLIT_K"] = str(args.force) - - q_int4, scales_gn, zeros_gn = _quantize_random(args.k, args.n, args.group, seed=0) - w_q, w_zp, w_s = _pack_weights(q_int4, scales_gn, zeros_gn, args.group) - x = (0.25 * torch.randn((args.m, args.k), device="cuda", dtype=torch.float32)).to( - torch.float16 - ) - w_q_t = torch.from_numpy(w_q).to("cuda") - w_zp_t = torch.from_numpy(w_zp).to("cuda") - w_s_t = torch.from_numpy(w_s).to("cuda") - g_idx = torch.empty(0, dtype=torch.int32, device="cuda") - use_v2 = bool(args.use_v2) - - def call(): - return torch.ops._rocm_C.gptq_gemm_rdna2_prefill( - x, w_q_t, w_zp_t, w_s_t, g_idx, use_v2 - ) - - for _ in range(3): - call() - torch.cuda.synchronize() - times = [] - for _ in range(args.iters): - s = torch.cuda.Event(enable_timing=True) - e = torch.cuda.Event(enable_timing=True) - s.record() - call() - e.record() - torch.cuda.synchronize() - times.append(s.elapsed_time(e)) - times.sort() - ms = times[len(times) // 2] - print(f"{args.m},{args.k},{args.n},{args.group},{args.force},{ms:.4f}") - - -if __name__ == "__main__": - main() diff --git a/tools/rdna2_028/probe_gptq_k.py b/tools/rdna2_028/probe_gptq_k.py deleted file mode 100644 index 70761b3b7eb9..000000000000 --- a/tools/rdna2_028/probe_gptq_k.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Focused probe: W4A16 prefill (gptq_gemm_rdna2_prefill) output norm vs k. - -Reproduces the k=4352 zero-output anomaly from probe_w4a8_op.py. Uses the -same numpy packing as probe_w4a8_op.py (no vLLM config needed). -""" -import numpy as np -import torch - -SHUFFLE_SLOTS = (0, 2, 4, 6, 1, 3, 5, 7) - - -def _pack_k_major(q_kn): - k, n = q_kn.shape - q = q_kn.astype(np.uint32).reshape(k // 8, 8, n) - out = np.zeros((k // 8, n), dtype=np.uint32) - for i in range(8): - out |= q[:, i, :] << np.uint32(4 * i) - return out - - -def _exllama_shuffle(packed): - packed = packed.astype(np.uint32) - out = np.zeros_like(packed) - for slot, k_off in enumerate(SHUFFLE_SLOTS): - nibble = (packed >> np.uint32(4 * k_off)) & np.uint32(0xF) - out |= nibble << np.uint32(4 * slot) - return out - - -def _pack_zeros_n_major(z_gn): - g, n = z_gn.shape - z = z_gn.astype(np.uint32).reshape(g, n // 8, 8) - out = np.zeros((g, n // 8), dtype=np.uint32) - for j in range(8): - out |= z[:, :, j] << np.uint32(4 * j) - return out - - -def main(): - m, n, g = 225, 5120, 32 - rng = np.random.default_rng(0) - for k in [1536, 2176, 2560, 4096, 4352, 5120, 8192]: - if k % g != 0 or k % 32 != 0: - continue - x = (0.25 * torch.randn((m, k), device="cuda", dtype=torch.float32)).to( - torch.float16 - ) - q_int4 = rng.integers(0, 16, size=(k, n), dtype=np.int32) - scales = (0.05 * rng.random((k // g, n)) + 0.01).astype(np.float32) - zeros = rng.integers(0, 16, size=(k // g, n), dtype=np.int32) - wq = torch.from_numpy( - _exllama_shuffle(_pack_k_major(q_int4)).astype(np.uint32) - ).to("cuda") - wzp = torch.from_numpy(_pack_zeros_n_major(zeros).astype(np.uint32)).to("cuda") - ws = torch.from_numpy(scales.astype(np.float16)).to("cuda") - gi = torch.empty(0, dtype=torch.int32, device="cuda") - out = torch.ops._rocm_C.gptq_gemm_rdna2_prefill(x, wq, wzp, ws, gi, False) - out8 = torch.ops._rocm_C.w4a8_gemm_rdna2(x, wq, wzp, ws, gi, False) - print( - f"k={k:5d} gptq norm={out.norm().item():10.4f} max={out.abs().max().item():8.4f} | " - f"w4a8 norm={out8.norm().item():10.4f} max={out8.abs().max().item():8.4f}" - ) - - -if __name__ == "__main__": - main() diff --git a/tools/rdna2_028/probe_inmodel_layer.py b/tools/rdna2_028/probe_inmodel_layer.py deleted file mode 100644 index 57371ed0a8f5..000000000000 --- a/tools/rdna2_028/probe_inmodel_layer.py +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""In-model layer test: real checkpoint tensors, production compressed-tensors -layout, processed by the actual RDNA2W4A16LinearKernel; W4A8 on vs off. - -This is the faithful path: it exercises the real AWQ qweight/qzeros/scales -layout the model produces, RDNA2W4A16LinearKernel.process_weights_after_loading -(the AWQ qzeros transpose included), and the W4A8 dispatcher + kernel. -Single GPU. -""" -import os -import sys - -import torch - -torch.manual_seed(0) - -from vllm.config import VllmConfig, set_current_vllm_config # noqa: E402 -from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E402 - MPLinearLayerConfig, -) -from vllm.model_executor.kernels.linear.mixed_precision.rdna2_w4a16 import ( # noqa: E402 - RDNA2W4A16LinearKernel, -) -from vllm.model_executor.parameter import ( # noqa: E402 - GroupQuantScaleParameter, - PackedvLLMParameter, -) -from vllm.scalar_type import scalar_types # noqa: E402 -from safetensors import safe_open # noqa: E402 - -MODEL = ( - "/home/chenco_adm/.cache/huggingface/hub/models--cyankiwi--Qwen3.8-27B-AWQ-INT4" - "/snapshots/63768c10df38c0395e12ef49edac1bd539eaeeea" -) -GROUP = 32 - - -def _find(name): - for i in range(1, 20): - fn = f"{MODEL}/model-{i:05d}-of-00005.safetensors" - if os.path.exists(fn): - with safe_open(fn, framework="pt") as f: - if name in f.keys(): - return f.get_tensor(name) - raise KeyError(name) - - -def _build_layer(layer_name): - wp = _find(layer_name + ".weight_packed").to(torch.int32).cuda() - ws = _find(layer_name + ".weight_scale").to(torch.float16).cuda() - wz = _find(layer_name + ".weight_zero_point").to(torch.int32).cuda() - wsh = _find(layer_name + ".weight_shape").cpu().numpy() - n, k = int(wsh[0]), int(wsh[1]) - - class DummyLayer(torch.nn.Module): - pass - - layer = DummyLayer() - layer.register_parameter( - "weight_packed", - PackedvLLMParameter( - data=wp, weight_loader=lambda *a, **kk: None, - input_dim=1, output_dim=0, packed_dim=1, packed_factor=8, - ), - ) - layer.register_parameter( - "weight_scale", - GroupQuantScaleParameter( - data=ws, weight_loader=lambda *a, **kk: None, - input_dim=1, output_dim=0, - ), - ) - layer.register_parameter( - "weight_zero_point", - PackedvLLMParameter( - data=wz, weight_loader=lambda *a, **kk: None, - input_dim=1, output_dim=0, packed_dim=0, packed_factor=8, - ), - ) - return layer, k, n - - -def _ensure_tp_group(vcfg): - from vllm.distributed.parallel_state import get_tp_group - - try: - get_tp_group() - return - except AssertionError: - pass - from vllm.distributed import ( - ensure_model_parallel_initialized, - init_distributed_environment, - ) - init_distributed_environment( - world_size=1, rank=0, - distributed_init_method="tcp://127.0.0.1:0", local_rank=0, - ) - ensure_model_parallel_initialized(1, 1) - - -def run_layer(layer_name, m): - _, k, n = _build_layer(layer_name) - x = (0.25 * torch.randn((m, k), device="cuda", dtype=torch.float32)).to( - torch.float16 - ) - config = MPLinearLayerConfig( - full_weight_shape=(k, n), partition_weight_shape=(k, n), - weight_type=scalar_types.uint4, act_type=torch.float16, - group_size=GROUP, zero_points=True, - ) - - def _apply(w4a8: bool): - os.environ.pop("VLLM_RDNA2_W4A8_SDOT4", None) - if w4a8: - os.environ["VLLM_RDNA2_W4A8_SDOT4"] = "1" - layer, _, _ = _build_layer(layer_name) # fresh params per arm - kernel = RDNA2W4A16LinearKernel( - config, w_q_param_name="weight_packed", - w_s_param_name="weight_scale", - w_zp_param_name="weight_zero_point", - ) - kernel.process_weights_after_loading(layer) - return kernel.apply_weights(layer, x) - - out_off = _apply(False) - out_on = _apply(True) - den = out_off.float().norm() - rel = float("nan") if den == 0 else ((out_on.float() - out_off.float()).norm() / den).item() - print( - f"[{layer_name.split('.')[-1]}] k={k} n={n} m={m}: " - f"w4a8_vs_w4a16 rel-L2 = {rel:.5f} (off norm={den.item():.1f})" - ) - - -def main(): - name = sys.argv[1] if len(sys.argv) > 1 else "model.language_model.layers.3.self_attn.o_proj" - m = int(sys.argv[2]) if len(sys.argv) > 2 else 2001 - vcfg = VllmConfig() - with set_current_vllm_config(vcfg): - _ensure_tp_group(vcfg) - run_layer(name, m) - - -if __name__ == "__main__": - main() diff --git a/tools/rdna2_028/probe_model_inspect2.py b/tools/rdna2_028/probe_model_inspect2.py deleted file mode 100644 index 93eef8af7f3a..000000000000 --- a/tools/rdna2_028/probe_model_inspect2.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Inspect the AWQ checkpoint tensor shapes/dtypes for W4A8 real-weight tests.""" -import json -import os -import struct -import sys - -MODEL = sys.argv[1] if len(sys.argv) > 1 else ( - "/home/chenco_adm/.cache/huggingface/hub/models--cyankiwi--Qwen3.8-27B-AWQ-INT4" - "/snapshots/63768c10df38c0395e12ef49edac1bd539eaeeea" -) - - -def headers(fn): - with open(fn, "rb") as f: - ln = struct.unpack(" -- \ - python tools/rdna2_028/probe_mtp_sampling.py -""" -from __future__ import annotations - -import sys -import time - -import torch - -sys.path.insert(0, "/home/chenco_adm/vllm-rdna-0.28.0") -from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import rejection_sample # noqa: E402 - -V = 248320 -NUM_SPEC = 2 -NUM_REQS = 1 -NUM_LOGITS = NUM_REQS * (NUM_SPEC + 1) # 3 -ITERS = int(sys.argv[1]) if len(sys.argv) > 1 else 200 -DEV = "cuda" - -torch.manual_seed(0) -target_logits = torch.randn(NUM_LOGITS, V, device=DEV, dtype=torch.float32) * 4 -draft_logits = torch.randn(NUM_REQS, NUM_SPEC, V, device=DEV, dtype=torch.float32) * 4 -draft_sampled = torch.randint(0, V, (NUM_LOGITS,), device=DEV, dtype=torch.int64) -cu_num_logits = torch.tensor([0, NUM_LOGITS], device=DEV, dtype=torch.int64) -pos = torch.arange(NUM_LOGITS, device=DEV, dtype=torch.int64) -idx_mapping = torch.tensor([0], device=DEV, dtype=torch.int64) -expanded_idx_mapping = torch.zeros(NUM_LOGITS, device=DEV, dtype=torch.int64) -expanded_local_pos = torch.tensor([0, 1, 2], device=DEV, dtype=torch.int64) -temperature = torch.tensor([0.0], device=DEV, dtype=torch.float32) -seed = torch.tensor([12345], device=DEV, dtype=torch.int64) - -# warmup / JIT compile -for _ in range(3): - rejection_sample( - target_logits, draft_logits, draft_sampled, cu_num_logits, pos, - idx_mapping, expanded_idx_mapping, expanded_local_pos, temperature, - seed, NUM_SPEC, - ) -torch.cuda.synchronize() - -t0 = time.perf_counter() -for _ in range(ITERS): - rejection_sample( - target_logits, draft_logits, draft_sampled, cu_num_logits, pos, - idx_mapping, expanded_idx_mapping, expanded_local_pos, temperature, - seed, NUM_SPEC, - ) -torch.cuda.synchronize() -t1 = time.perf_counter() -print(f"MTP trio+insert: {ITERS} iters in {t1 - t0:.3f}s -> {(t1 - t0) / ITERS * 1e6:.1f} us/call") diff --git a/tools/rdna2_028/probe_prefill_split.py b/tools/rdna2_028/probe_prefill_split.py deleted file mode 100644 index 6bf82544bf43..000000000000 --- a/tools/rdna2_028/probe_prefill_split.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Op-level latency of the dense W4A16 prefill GEMM per production shape. - -Times `_rocm_C.gptq_gemm_rdna2_prefill` (the W4A16 arm the W4A8 entry falls -back to) and reports the output norm/NaN flag. Run once per kernel build; with -VLLM_RDNA2_PREFILL_DEBUG=1 the kernel prints its chosen split on stdout -(`[rdna2_prefill_split] m=... n=... k=... split=...`), which is joined with the -latencies afterwards to compare the split choice across builds. - -The per-shape buffers are byte-identical to what RDNA2W4A16LinearKernel leaves -behind (same packing as probe_w4a8_op.py). Single GPU. -""" - -import argparse -import os -import sys - -import torch - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from probe_w4a8_op import ( # noqa: E402 - _pack_weights, - _quantize_random, -) - -PROD_SHAPES = [ - (m, k, n, 32) - for m in (225, 2001, 2048) - for k in (1536, 4352, 5120, 6144, 8704) - for n in (3584, 4096, 5120, 8704) -] - - -def _bench(fn, warmup=3, iters=30): - for _ in range(warmup): - fn() - torch.cuda.synchronize() - times = [] - for _ in range(iters): - s = torch.cuda.Event(enable_timing=True) - e = torch.cuda.Event(enable_timing=True) - s.record() - fn() - e.record() - torch.cuda.synchronize() - times.append(s.elapsed_time(e)) - times.sort() - return times[len(times) // 2] - - -def run_shape(m, k, n, group, use_v2, iters): - q_int4, scales_gn, zeros_gn = _quantize_random(k, n, group, seed=0) - w_q, w_zp, w_s = _pack_weights(q_int4, scales_gn, zeros_gn, group) - x = (0.25 * torch.randn((m, k), device="cuda", dtype=torch.float32)).to( - torch.float16 - ) - w_q_t = torch.from_numpy(w_q).to("cuda") - w_zp_t = torch.from_numpy(w_zp).to("cuda") - w_s_t = torch.from_numpy(w_s).to("cuda") - g_idx = torch.empty(0, dtype=torch.int32, device="cuda") - - def call(): - return torch.ops._rocm_C.gptq_gemm_rdna2_prefill( - x, w_q_t, w_zp_t, w_s_t, g_idx, use_v2 - ) - - ms = _bench(call, iters=iters) - out = call() - norm = float(out.to(torch.float32).norm()) - has_nan = bool(torch.isnan(out).any()) - return ms, norm, has_nan - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--out", required=True, help="CSV output path") - ap.add_argument("--shapes", default=None, help="m,k,n,group;... override") - ap.add_argument("--variants", default="awq,gptq") - ap.add_argument("--iters", type=int, default=30) - args = ap.parse_args() - - if args.shapes: - shapes = [tuple(int(x) for x in s.split(",")) for s in args.shapes.split(";")] - else: - shapes = PROD_SHAPES - - variants = { - "awq": (True, False), # uint4: use_v2_format=True - "gptq": (False, True), # uint4b8: use_v2_format=False - } - chosen = [v for v in args.variants.split(",") if v in variants] - - with open(args.out, "w") as f: - f.write("m,k,n,group,variant,ms,norm,nan\n") - for (m, k, n, group) in shapes: - for v in chosen: - use_v2, _gptq = variants[v] - try: - ms, norm, has_nan = run_shape(m, k, n, group, use_v2, args.iters) - except Exception as e: # noqa: BLE001 - f.write(f"{m},{k},{n},{group},{v},ERROR,0,1 # {e!r}\n") - f.flush() - continue - f.write(f"{m},{k},{n},{group},{v},{ms:.4f},{norm:.4f},{int(has_nan)}\n") - f.flush() - print(f"({m},{k},{n},{group}) {v} {ms:.4f}ms norm={norm:.4f} nan={has_nan}") - - -if __name__ == "__main__": - main() diff --git a/tools/rdna2_028/probe_real_weights.py b/tools/rdna2_028/probe_real_weights.py deleted file mode 100644 index edb7529d37db..000000000000 --- a/tools/rdna2_028/probe_real_weights.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Real-weight W4A8 vs W4A16 op test on the AWQ 27B checkpoint. - -Loads actual compressed-tensors AWQ weights, converts to the kernel layout -(qweight [K/8,N] shuffled, qzeros [G,N/8], scales [G,N]), and compares -w4a8_gemm_rdna2 vs gptq_gemm_rdna2_prefill vs an fp32 dequant reference. -Decisive test for a *data-dependent* W4A8 bug that random weights miss. -""" -import os - -import numpy as np -import torch -from safetensors import safe_open - -SHUFFLE_SLOTS = (0, 2, 4, 6, 1, 3, 5, 7) -GROUP = 32 -MODEL = ( - "/home/chenco_adm/.cache/huggingface/hub/models--cyankiwi--Qwen3.8-27B-AWQ-INT4" - "/snapshots/63768c10df38c0395e12ef49edac1bd539eaeeea" -) - - -def _find(name): - for i in range(1, 20): - fn = f"{MODEL}/model-{i:05d}-of-00005.safetensors" - if os.path.exists(fn): - with safe_open(fn, framework="pt") as f: - if name in f.keys(): - return f.get_tensor(name) - raise KeyError(name) - - -def _pack_k_major(q_kn): - k, n = q_kn.shape - q = q_kn.astype(np.uint32).reshape(k // 8, 8, n) - out = np.zeros((k // 8, n), dtype=np.uint32) - for i in range(8): - out |= q[:, i, :] << np.uint32(4 * i) - return out - - -def _exllama_shuffle(packed): - packed = packed.astype(np.uint32) - out = np.zeros_like(packed) - for slot, k_off in enumerate(SHUFFLE_SLOTS): - nibble = (packed >> np.uint32(4 * k_off)) & np.uint32(0xF) - out |= nibble << np.uint32(4 * slot) - return out - - -def _pack_zeros_n_major(z_gn): - g, n = z_gn.shape - z = z_gn.astype(np.uint32).reshape(g, n // 8, 8) - out = np.zeros((g, n // 8), dtype=np.uint32) - for j in range(8): - out |= z[:, :, j] << np.uint32(4 * j) - return out - - -def _unpack_q(wp, k, n): - """wp [N, K/8] -> qkn [K, N].""" - qkn = np.zeros((k, n), dtype=np.int64) - for i in range(8): - qkn[i::8, :] = (wp >> np.uint32(4 * i)).T & 0xF - return qkn - - -def _unpack_z(wz, n, g): - """wz [N/8, G] packed -> zng [N, G].""" - zng = np.zeros((n, g), dtype=np.int64) - for j in range(8): - zng[j::8, :] = (wz >> np.uint32(4 * j)) & 0xF - return zng - - -def test_layer(layer_name, k_shard=None, m=2001, label=""): - wp = _find(layer_name + ".weight_packed").to(torch.int64).cpu().numpy() # [N, K/8] - ws = _find(layer_name + ".weight_scale").to(torch.float32).cpu().numpy() # [N, G] - wz = _find(layer_name + ".weight_zero_point").to(torch.int64).cpu().numpy() # [N/8, G] - wsh = _find(layer_name + ".weight_shape").cpu().numpy() - n, k = int(wsh[0]), int(wsh[1]) - assert k % GROUP == 0 - - if k_shard is not None: - sh = k_shard - wp = wp[:, 0 : sh // 8] - ws = ws[:, 0 : sh // GROUP] - wz = wz[:, 0 : sh // GROUP] - k = sh - - qkn = _unpack_q(wp, k, n) - zng = _unpack_z(wz, n, k // GROUP) - s_full = np.repeat(ws, GROUP, axis=1).T # [K, N] - z_full = np.repeat(zng, GROUP, axis=1).T # [K, N] - x_np = (0.25 * np.random.default_rng(0).standard_normal((m, k))).astype(np.float32) - x = torch.from_numpy(x_np).to(torch.float16).cuda() - - for tag, zoff in (("lit/off0", 0), ("gptq/off1", 1)): - w = (qkn.astype(np.float32) - (z_full.astype(np.float32) + zoff)) * s_full.astype(np.float32) - ref = torch.from_numpy(x_np @ w).cuda() - - wq = torch.from_numpy( - _exllama_shuffle(_pack_k_major(qkn.astype(np.int32))).astype(np.uint32) - ).cuda() - wzp = torch.from_numpy(_pack_zeros_n_major(zng.T.astype(np.int32)).astype(np.uint32)).cuda() - wsc = torch.from_numpy(ws.T.astype(np.float16).copy()).cuda() # [G, N] - gi = torch.empty(0, dtype=torch.int32, device="cuda") - - use_v2 = tag.startswith("lit") - out8 = torch.ops._rocm_C.w4a8_gemm_rdna2(x, wq, wzp, wsc, gi, use_v2) - outg = torch.ops._rocm_C.gptq_gemm_rdna2_prefill(x, wq, wzp, wsc, gi, use_v2) - - def rl(a, b): - d = b.float().norm() - return float("nan") if d == 0 else ((a.float() - b.float()).norm() / d).item() - - print( - f"[{label}{layer_name.split('.')[-1]}] k={k} n={n} m={m} zero={tag}: " - f"w4a8vref={rl(out8, ref):.5f} gptqvref={rl(outg, ref):.5f} " - f"w4a8vgptq={rl(out8, outg):.5f}" - ) - - -def main(): - torch.manual_seed(0) - test_layer("model.language_model.layers.3.self_attn.o_proj", label="FULL ") - test_layer("model.language_model.layers.3.mlp.down_proj", k_shard=4352, label="SHARD") - - -if __name__ == "__main__": - main() diff --git a/tools/rdna2_028/probe_w4a8.py b/tools/rdna2_028/probe_w4a8.py deleted file mode 100644 index fda765040d69..000000000000 --- a/tools/rdna2_028/probe_w4a8.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Coherence + garbage probe for the W4A8 A/B (Qwen3.8-27B-AWQ-INT4). - -Sends deterministic prompts (temp=0) to a running server and reports, per -prompt, the raw text plus a coherence/garbage verdict. Used to prove the -W4A8=0 arm is coherent and to detect the W4A8=1 arm's collapse. - -Usage: - python probe_w4a8.py [n_repeats] -""" -import json -import re -import sys - -from openai import OpenAI - -BASE_URL = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:18210/v1" -MODEL = sys.argv[2] if len(sys.argv) > 2 else "q27d" -N = int(sys.argv[3]) if len(sys.argv) > 3 else 1 - -PROMPTS = [ - ("france", "The capital of France is", ["paris"]), - ("math", "2 + 2 =", ["4"]), - ("gpu", "In one sentence, what is a GPU?", []), - ("python", "Write one sentence about Python.", []), -] - -# A token repeated >= this many times in a row is a collapse signature. -REPEAT_THRESHOLD = 5 -# If one whitespace token accounts for >= this share of the output, the model -# has degenerated onto a single token (e.g. "Register Register ..." or a run -# like "ductductduct" that has no whitespace at all). -DOMINANCE_THRESHOLD = 0.5 -# Longest repeated substring (in chars) that signals a degenerate loop even -# without whitespace separators. -LOOP_MIN_LEN = 6 - - -def _longest_repeat_span(text: str) -> int: - """Return the longest run length of the repeated form s = unit * k, k >= 3.""" - n = len(text) - best = 0 - for unit in range(1, max(2, n // 3 + 1)): - if n % unit: - continue - rep = n // unit - if rep < 3: - continue - if text == text[:unit] * rep: - best = max(best, unit) - return best - - -def garbage_flags(text: str) -> list[str]: - flags = [] - if not text.strip(): - flags.append("empty") - toks = re.findall(r"\S+", text) - if toks: - top = max(set(toks), key=toks.count) - if toks.count(top) / len(toks) >= DOMINANCE_THRESHOLD and len(toks) >= 4: - flags.append(f"dominant:{top[:24]}") - # repeated token collapse: same whitespace-delimited token >= threshold - run = 1 - for i in range(1, len(toks)): - if toks[i] == toks[i - 1]: - run += 1 - if run >= REPEAT_THRESHOLD: - flags.append(f"repeat:{toks[i][:24]}") - break - else: - run = 1 - # whitespace-free degenerate loop, e.g. "ductductduct..." - span = _longest_repeat_span(text.strip()) - if span >= LOOP_MIN_LEN: - flags.append(f"char-loop:{text.strip()[:span][:24]}") - # non-printable / replacement chars - if re.search(r"[\ufffd]", text): - flags.append("replacement-char") - return flags - - -def main() -> None: - client = OpenAI(base_url=BASE_URL, api_key="dummy") - results = [] - for _ in range(N): - for name, prompt, expect in PROMPTS: - try: - r = client.completions.create( - model=MODEL, - prompt=prompt, - max_tokens=32, - temperature=0.0, - extra_body={"ignore_eos": True}, - ) - text = r.choices[0].text or "" - except Exception as e: # noqa: BLE001 - text = "" - results.append({"name": name, "prompt": prompt, "error": repr(e)}) - print(f"[{name}] ERROR: {e!r}") - continue - flags = garbage_flags(text) - low = text.lower() - coherent = any(e in low for e in expect) if expect else bool(text.strip()) - verdict = "OK" if (coherent and not flags) else "BAD" - results.append( - { - "name": name, - "prompt": prompt, - "output": text, - "coherent": coherent, - "garbage": flags, - "verdict": verdict, - } - ) - print(f"[{name}] {verdict:3s} coherent={coherent} garbage={flags} | {text[:120]!r}") - - ok = sum(1 for r in results if r.get("verdict") == "OK") - bad = sum(1 for r in results if r.get("verdict") == "BAD") - print(f"\n=== {ok} OK / {bad} BAD / {len(results)} total ===") - sys.exit(0 if bad == 0 else 1) - - -if __name__ == "__main__": - main() diff --git a/tools/rdna2_028/probe_w4a8_moe.py b/tools/rdna2_028/probe_w4a8_moe.py deleted file mode 100644 index 5b380fe03915..000000000000 --- a/tools/rdna2_028/probe_w4a8_moe.py +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Per-expert micro-benchmark: RDNA2 fused MoE W4A8 (sdot4) vs W4A16. - -Runs the two ``torch.ops._rocm_C`` MoE entries on the Qwen3.8-Flash-Next AWQ -expert geometry (E=128 local, top_k=10, group_size=128) over a token sweep and -reports ms/call plus the W4A8/W4A16 speedup. A rel-L2 sanity check against the -W4A16 output guards against a fast-but-wrong kernel. - -Usage: - python probe_w4a8_moe.py [--experts 128] [--group 128] [--top-k 10] - [--iters 50] [--tokens 1,64,256,1024,2048] -""" - -import argparse -import time - -import torch - -from vllm import _custom_ops as ops -from vllm.model_executor.layers.fused_moe.moe_align_block_size import ( - moe_align_block_size, -) -from vllm.model_executor.layers.quantization.utils.quant_utils import ( - pack_quantized_values_into_int32, -) -from vllm.scalar_type import scalar_types - -device = "cuda" - - -def _packed_weights(e, k, n): - q = torch.randint(0, 16, (e, k, n), dtype=torch.int32, device=device) - packed = torch.zeros(e, k // 8, n, dtype=torch.int32, device=device) - for i in range(8): - packed |= (q[:, i::8, :] & 0xF) << (i * 4) - for expert in range(e): - we = packed[expert].contiguous() - ops.gptq_shuffle(we, 4) - packed[expert] = we - return packed - - -def _scales(e, groups, n): - return (0.05 * torch.rand((e, groups, n), device=device) + 0.01).to( - torch.float16 - ) - - -def _qzeros(e, groups, n): - zeros = torch.full( - (groups, n), - scalar_types.uint4b8.bias - 1, - dtype=torch.int32, - device=device, - ) - packed = pack_quantized_values_into_int32( - zeros, scalar_types.uint4b8, packed_dim=1 - ) - return packed.unsqueeze(0).expand(e, -1, -1).contiguous() - - -def _make_case(e, k, n, group, tokens, top_k, block_size_m, seed): - torch.manual_seed(seed) - a = torch.randn(tokens, k, dtype=torch.float16, device=device) - w = _packed_weights(e, k, n) - s = _scales(e, k // group, n) - z = _qzeros(e, k // group, n) - ids = torch.randint(0, e, (tokens, top_k), device=device, dtype=torch.int32) - si, ei, ntp = moe_align_block_size(ids, block_size_m, e) - return a, w, s, z, si, ei, ntp - - -def _time(fn, iters): - for _ in range(5): - fn() - torch.cuda.synchronize() - start = torch.cuda.Event(enable_timing=True) - end = torch.cuda.Event(enable_timing=True) - start.record() - for _ in range(iters): - fn() - end.record() - torch.cuda.synchronize() - return start.elapsed_time(end) / iters - - -def _run(kind, a, c, w, s, z, si, ei, ntp, top_k, block_size_m, mul_topk, - output_topk): - if kind == "w4a8": - ops.moe_w4a8_gemm_rdna2( - a, c, w, s, z, torch.empty(0, device=device), si, ei, ntp, top_k, - block_size_m, mul_topk, output_topk, False, - ) - else: - ops.moe_gptq_gemm_rdna2( - a, c, w, s, z, torch.empty(0, device=device), si, ei, ntp, top_k, - block_size_m, mul_topk, output_topk, - ) - - -def _rel_l2(got, ref): - g, r = got.float(), ref.float() - return ((g - r).norm() / r.norm()).item() - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--experts", type=int, default=128) - ap.add_argument("--group", type=int, default=128) - ap.add_argument("--top-k", type=int, default=10) - ap.add_argument("--block-size-m", type=int, default=8) - ap.add_argument("--iters", type=int, default=50) - ap.add_argument("--tokens", default="1,64,256,1024,2048") - args = ap.parse_args() - - tokens_list = [int(t) for t in args.tokens.split(",")] - # Flash-Next w1 (gate+up) and w2 (down) expert shapes. - passes = [ - ("w13", 2560, 1280), - ("w2", 640, 2560), - ] - print( - f"E={args.experts} group={args.group} top_k={args.top_k} " - f"bsm={args.block_size_m} iters={args.iters}" - ) - for name, k, n in passes: - print(f"\n== {name}: K={k} N={n} ==") - print(f"{'tokens':>8} {'w4a16 ms':>10} {'w4a8 ms':>10} " - f"{'speedup':>8} {'relL2':>9}") - for tokens in tokens_list: - params = _make_case( - args.experts, k, n, args.group, tokens, args.top_k, - args.block_size_m, seed=1000 + tokens, - ) - a, w, s, z, si, ei, ntp = params - flat = tokens * args.top_k - c16 = torch.zeros(flat, n, dtype=torch.float16, device=device) - c8 = torch.zeros(flat, n, dtype=torch.float16, device=device) - - _run("w4a16", a, c16, w, s, z, si, ei, ntp, args.top_k, - args.block_size_m, False, 0) - torch.cuda.synchronize() - ref = c16.clone() - - def run16(): - c16.zero_() - _run("w4a16", a, c16, w, s, z, si, ei, ntp, args.top_k, - args.block_size_m, False, 0) - - def run8(): - c8.zero_() - _run("w4a8", a, c8, w, s, z, si, ei, ntp, args.top_k, - args.block_size_m, False, 0) - - ms16 = _time(run16, args.iters) - ms8 = _time(run8, args.iters) - - c8.zero_() - _run("w4a8", a, c8, w, s, z, si, ei, ntp, args.top_k, - args.block_size_m, False, 0) - torch.cuda.synchronize() - rel = _rel_l2(c8, ref) - - print(f"{tokens:>8} {ms16:>10.3f} {ms8:>10.3f} " - f"{ms16 / ms8:>7.2f}x {rel:>9.5f}") - print(f"\n[done {time.strftime('%H:%M:%S')}]") - - -if __name__ == "__main__": - main() diff --git a/tools/rdna2_028/probe_w4a8_op.py b/tools/rdna2_028/probe_w4a8_op.py deleted file mode 100644 index 6d9fbe408f43..000000000000 --- a/tools/rdna2_028/probe_w4a8_op.py +++ /dev/null @@ -1,220 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Op-level W4A8 sdot4 GEMM correctness sweep (single GPU, gfx1030). - -For each (m, k, n, group) shape, packs an IDENTICAL int4 weight buffer and -compares the W4A8 fast path (`w4a8_gemm_rdna2`) against the W4A16 prefill -reference (`gptq_gemm_rdna2_prefill`) via rel-L2, and against a true fp32 -dequant reference to classify which of the two is wrong. - -Runs BOTH quant variants on the same shape: - * GPTQ ``uint4b8`` -> use_v2_format=False (zero_offset = 1, kernel adds 1) - * AWQ ``uint4`` -> use_v2_format=True (zero_offset = 0, literal zero) - -The weight buffer the kernels read is byte-identical to what -RDNA2W4A16LinearKernel leaves behind (pack_k_major + gptq_shuffle for W, -pack_n_major for qzeros). This is the production apples-to-apples -comparison. Modes: - - --mode shapes run the given (or default in-model) shape set - --mode bisect-k fix n,group; sweep k over [k_lo, k_hi) step k_step - --mode bisect-group fix m,n,k; sweep group over a list - -Usage: - python probe_w4a8_op.py --mode shapes - python probe_w4a8_op.py --mode bisect-k --n 5120 --group 32 --k-lo 512 --k-hi 6144 --k-step 32 - python probe_w4a8_op.py --mode bisect-group --m 2001 --n 5120 --k 4352 --groups 32,64,128 -""" -import argparse -import sys - -import numpy as np -import torch - -# The full distinct fast-path shape set extracted from the serve logs -# (grep VLLM_RDNA2_W4A8 /home/chenco_adm/w4a8_runs/*/serve.log). All group=32. -DEFAULT_SHAPES = [ - (225, 1536, 5120, 32), - (225, 4352, 5120, 32), - (225, 5120, 3584, 32), - (225, 5120, 4096, 32), - (225, 5120, 8704, 32), - (2001, 1536, 5120, 32), - (2001, 4352, 5120, 32), - (2001, 5120, 3584, 32), - (2001, 5120, 4096, 32), - (2001, 5120, 8704, 32), - (2048, 1536, 5120, 32), - (2048, 4352, 5120, 32), - (2048, 5120, 3584, 32), - (2048, 5120, 4096, 32), - (2048, 5120, 8704, 32), -] - -# gptq_shuffle slot permutation (from explore/reference.py, matches the .cuh). -SHUFFLE_SLOTS = (0, 2, 4, 6, 1, 3, 5, 7) -# kAPerm for the A tile (matches csrc/rocm/explore/w4a8_sdot4.cuh). -A_PERM = (0, 4, 1, 5, 2, 6, 3, 7) -MTILE = 8 - - -def _pack_k_major(q_kn: np.ndarray) -> np.ndarray: - """[K, N] int4 -> [K/8, N] uint32, nibble i = q[k0+i, n] << 4i.""" - k, n = q_kn.shape - q = q_kn.astype(np.uint32).reshape(k // 8, 8, n) - out = np.zeros((k // 8, n), dtype=np.uint32) - for i in range(8): - out |= q[:, i, :] << np.uint32(4 * i) - return out - - -def _exllama_shuffle(packed: np.ndarray) -> np.ndarray: - packed = packed.astype(np.uint32) - out = np.zeros_like(packed) - for slot, k_off in enumerate(SHUFFLE_SLOTS): - nibble = (packed >> np.uint32(4 * k_off)) & np.uint32(0xF) - out |= nibble << np.uint32(4 * slot) - return out - - -def _pack_zeros_n_major(z_gn: np.ndarray) -> np.ndarray: - """[G, N] int4 -> [G, N/8] uint32, nibble j = z[g, n0+j] << 4j.""" - g, n = z_gn.shape - z = z_gn.astype(np.uint32).reshape(g, n // 8, 8) - out = np.zeros((g, n // 8), dtype=np.uint32) - for j in range(8): - out |= z[:, :, j] << np.uint32(4 * j) - return out - - -def _quantize_random(k: int, n: int, group: int, seed: int): - """Deterministic random int4 weights, fp16 scales, int32 zeros.""" - rng = np.random.default_rng(seed) - g = k // group - q_int4 = rng.integers(0, 16, size=(k, n), dtype=np.int32) - scales = (0.05 * rng.random((g, n)) + 0.01).astype(np.float32) - zeros = rng.integers(0, 16, size=(g, n), dtype=np.int32) - return q_int4, scales, zeros - - -def _pack_weights(q_int4, scales_gn, zeros_gn, group): - """Produce the exact buffers the kernels consume.""" - w_q = _exllama_shuffle(_pack_k_major(q_int4)).astype(np.uint32) - w_zp = _pack_zeros_n_major(zeros_gn).astype(np.uint32) - w_s = scales_gn.astype(np.float16) - return w_q, w_zp, w_s - - -def _fp32_reference(x_mk, q_int4, scales_gn, zeros_gn, group, gptq): - """True dequant reference: w = (q - z) * scale, out = x @ w.""" - k, n = q_int4.shape - s_full = np.repeat(scales_gn, group, axis=0).astype(np.float32) - z_full = np.repeat(zeros_gn, group, axis=0).astype(np.float32) - if gptq: - z_full = z_full + 1.0 # GPTQv1: stored zero = actual - 1 - w_fp = (q_int4.astype(np.float32) - z_full) * s_full - return x_mk.astype(np.float32) @ w_fp - - -def run_shape(m, k, n, group, seed=0): - """Return dict with rel-L2 metrics for one (m,k,n,group), both variants.""" - q_int4, scales_gn, zeros_gn = _quantize_random(k, n, group, seed) - w_q, w_zp, w_s = _pack_weights(q_int4, scales_gn, zeros_gn, group) - x_mk = (0.25 * torch.randn((m, k), device="cuda", dtype=torch.float32)).to( - torch.float16 - ) - w_q_t = torch.from_numpy(w_q).to("cuda") - w_zp_t = torch.from_numpy(w_zp).to("cuda") - w_s_t = torch.from_numpy(w_s).to("cuda") - g_idx = torch.empty(0, dtype=torch.int32, device="cuda") - - res = {"shape": (m, k, n, group)} - for variant, use_v2, gptq in (("AWQ", True, False), ("GPTQ", False, True)): - out_w4a8 = torch.ops._rocm_C.w4a8_gemm_rdna2( - x_mk, w_q_t, w_zp_t, w_s_t, g_idx, use_v2 - ) - out_gptq = torch.ops._rocm_C.gptq_gemm_rdna2_prefill( - x_mk, w_q_t, w_zp_t, w_s_t, g_idx, use_v2 - ) - ref = torch.from_numpy( - _fp32_reference( - x_mk.cpu().numpy().astype(np.float32), - q_int4, - scales_gn, - zeros_gn, - group, - gptq, - ) - ).to("cuda") - - def rel_l2(a, b): - den = b.to(torch.float32).norm() - if den.item() == 0.0: - return float("nan") - return ((a.to(torch.float32) - b.to(torch.float32)).norm() / den).item() - - res[f"{variant}_w4a8_vs_gptq"] = rel_l2(out_w4a8, out_gptq) - res[f"{variant}_w4a8_vs_ref"] = rel_l2(out_w4a8, ref) - res[f"{variant}_gptq_vs_ref"] = rel_l2(out_gptq, ref) - return res - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--mode", default="shapes", choices=["shapes", "bisect-k", "bisect-group"]) - ap.add_argument("--m", type=int, default=2001) - ap.add_argument("--n", type=int, default=5120) - ap.add_argument("--group", type=int, default=32) - ap.add_argument("--k-lo", type=int, default=512) - ap.add_argument("--k-hi", type=int, default=6144) - ap.add_argument("--k-step", type=int, default=32) - ap.add_argument("--k", type=int, default=4352) - ap.add_argument("--groups", default="32,64,128") - ap.add_argument("--seed", type=int, default=0) - ap.add_argument("--shapes", default=None, help="comma list of m,k,n,group") - args = ap.parse_args() - - if args.mode == "shapes": - shapes = DEFAULT_SHAPES - if args.shapes: - shapes = [ - tuple(int(x) for x in s.split(",")) - for s in args.shapes.split(";") - ] - elif args.mode == "bisect-k": - shapes = [ - (args.m, kk, args.n, args.group) - for kk in range(args.k_lo, args.k_hi, args.k_step) - if kk % 32 == 0 and kk % args.group == 0 - ] - else: # bisect-group - groups = [int(x) for x in args.groups.split(",")] - shapes = [ - (args.m, args.k, args.n, g) - for g in groups - if args.k % g == 0 - ] - - hdr = f"{'shape':>22} | {'AWQ w4a8/gptq':>14} | {'AWQ w4a8/ref':>13} | {'AWQ gptq/ref':>13} | {'GPTQ w4a8/gptq':>15} | {'GPTQ w4a8/ref':>14} | {'GPTQ gptq/ref':>14}" - print(hdr) - print("-" * len(hdr)) - for (m, k, n, group) in shapes: - try: - r = run_shape(m, k, n, group, seed=args.seed) - except Exception as e: # noqa: BLE001 - print(f"{f'({m},{k},{n},{group})':>22} | ERROR: {e!r}") - continue - def fmt(v): - return f"{v:.4f}" if v == v else " nan" - print( - f"{f'({m},{k},{n},{group})':>22} | {fmt(r['AWQ_w4a8_vs_gptq']):>14} | " - f"{fmt(r['AWQ_w4a8_vs_ref']):>13} | {fmt(r['AWQ_gptq_vs_ref']):>13} | " - f"{fmt(r['GPTQ_w4a8_vs_gptq']):>15} | {fmt(r['GPTQ_w4a8_vs_ref']):>14} | " - f"{fmt(r['GPTQ_gptq_vs_ref']):>14}" - ) - sys.stdout.flush() - - -if __name__ == "__main__": - main() diff --git a/tools/rdna2_028/step0_parse.py b/tools/rdna2_028/step0_parse.py deleted file mode 100644 index ab0ba716ed91..000000000000 --- a/tools/rdna2_028/step0_parse.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Aggregate a rocprofv3 kernel_trace.csv into a kernel-share table. - -usage: step0_parse.py [--json out.json] - -Share = kernel busy time / total kernel dispatch time in the file. For a -clean-exit `bench throughput` run the file covers load + compile + the -measured prefill/decode, so treat the table as a ranking of where GPU time -goes, not a per-phase breakdown. -""" -from __future__ import annotations - -import csv -import json -import re -import sys -from collections import defaultdict - -GROUPS = [ - ("mtp_stats", re.compile(r"compute_local_logits_stats", re.I)), - ("mtp_reject", re.compile(r"rejection_kernel", re.I)), - ("mtp_resample", re.compile(r"resample_kernel", re.I)), - ("qsa", re.compile(r"qsa|sparse_paged_gqa|expand_qsa|compress_qsa", re.I)), - ("causal_conv1d_triton", re.compile(r"causal_conv1d_fwd_kernel|causal_conv1d_update_kernel", re.I)), - ("causal_conv1d_hip", re.compile(r"causal_conv1d_(fwd|update)_rdna2", re.I)), - ("fa_rdna2", re.compile(r"fa_rdna2", re.I)), - ("triton", re.compile(r"triton_", re.I)), - ("rocblas", re.compile(r"rocblas|Cijk_|gemm|Gemm|GEMM", re.I)), - ("dot2_w4a16", re.compile(r"gptq_gemm_rdna2|moe_gptq_gemm_rdna2|w4a8|dot2", re.I)), -] - - -def group_of(name: str) -> str: - for g, rx in GROUPS: - if rx.search(name): - return g - return "other" - - -def main() -> int: - path = sys.argv[1] - rows = [] - with open(path, newline="") as fh: - rd = csv.DictReader(fh) - for r in rd: - if r.get("Kind") != "KERNEL_DISPATCH": - continue - try: - dur = int(r["End_Timestamp"]) - int(r["Start_Timestamp"]) - except (KeyError, ValueError): - continue - rows.append((r.get("Kernel_Name", ""), dur)) - - per_kernel: dict[str, list[int]] = defaultdict(lambda: [0, 0]) - per_group: dict[str, list[int]] = defaultdict(lambda: [0, 0]) - total = 0 - for name, dur in rows: - if dur < 0: - continue - per_kernel[name][0] += dur - per_kernel[name][1] += 1 - per_group[group_of(name)][0] += dur - per_group[group_of(name)][1] += 1 - total += dur - - def fmt(d: int) -> str: - return f"{d/1e6:,.1f}" - - print(f"# {path}") - print(f"# dispatches={len(rows)} total_kernel_busy_ms={fmt(total)}") - print() - print("## by group") - print(f"{'group':26} {'busy_ms':>14} {'share':>7} {'dispatches':>11}") - for g, (d, n) in sorted(per_group.items(), key=lambda kv: -kv[1][0]): - print(f"{g:26} {fmt(d):>14} {100*d/total:6.1f}% {n:>11}") - print() - print("## top kernels") - print(f"{'busy_ms':>14} {'share':>7} {'n':>9} kernel") - for name, (d, n) in sorted(per_kernel.items(), key=lambda kv: -kv[1][0])[:40]: - short = name if len(name) <= 110 else name[:107] + "..." - print(f"{fmt(d):>14} {100*d/total:6.1f}% {n:>9} {short}") - - if "--json" in sys.argv: - out = sys.argv[sys.argv.index("--json") + 1] - with open(out, "w") as fh: - json.dump( - { - "total_ms": total / 1e6, - "dispatches": len(rows), - "groups": {g: {"ms": d / 1e6, "n": n} for g, (d, n) in per_group.items()}, - "kernels": {k: {"ms": d / 1e6, "n": n} for k, (d, n) in per_kernel.items()}, - }, - fh, - indent=2, - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/rdna2_028/step0_profile.sh b/tools/rdna2_028/step0_profile.sh deleted file mode 100644 index 7577d8f3af67..000000000000 --- a/tools/rdna2_028/step0_profile.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# STEP-0 in-model kernel profile for the gfx1030 fork. -# -# rocprofv3 in --run (spawn) mode around `vllm bench throughput`, which drives -# the real vLLM engine with the serving config and then exits cleanly so the -# rocprofiler tool finalizes and merges its output (the long-lived `vllm serve` -# path cannot be flushed: SIGINT does not exit vLLM and SIGKILL aborts the -# finalizer). One cell per invocation, one engine at a time. -# -# MTP=0 TAG=step0_mtp0 bash tools/rdna2_028/step0_profile.sh -# MTP=2 TAG=step0_mtp2 bash tools/rdna2_028/step0_profile.sh -# -# Cells: c=1 1k prefills + 128 decode steps, c=1 16k prefills + 128 decode -# steps. No /tmp (rocprof TMPDIR + all artifacts under $OUT). -set -uo pipefail - -V=${V:-/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0} -T=${T:-/home/chenco_adm/vllm-rdna-0.28.0} -MODEL=${MODEL:-/home/chenco_adm/hfcache/hub/models--wtdcode--Qwen3.8-Flash-Next-AWQ-W4A16/snapshots/0939125b929543a783ce700c90e36dd1a575c00c} -PLE=${PLE:-/home/chenco_adm/hfcache/hub/models--primitive-ai--Qwen3.8-Flash-Next-PLE-quant/snapshots/4f861b63f69e61bfc2e22130ec91ec67f03ec43e/ples_int4} -MTP=${MTP:-0} -OUTLEN=${OUTLEN:-128} -NUM_PROMPTS=${NUM_PROMPTS:-1} -TAG=${TAG:-step0_mtp$MTP} -OUT=${OUT:-/home/chenco_adm/w4a8_runs/$TAG} -CELLS=${CELLS:-"1024 16384"} -READY_SO=${READY_SO:-} - -mkdir -p "$OUT" -DRIVER_LOG=$OUT/driver.log -log(){ echo "[$(date +%H:%M:%S)] $*" | tee -a "$DRIVER_LOG"; } - -# --- ROCm/venv library resolution (mirror of rdna_launcher_common) ---------- -RSL=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -RS=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$RSL:$RS/host-math/lib:$RS/rocm_sysdeps/lib:$RS/core/lib:$V/lib/python3.12/site-packages/torch/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -export ROCM_HOME=${ROCM_HOME:-/opt/rocm/core-7.14} -export ROCM_PATH=${ROCM_PATH:-/opt/rocm/core-7.14} -export HIP_PATH=${HIP_PATH:-/opt/rocm/core-7.14} -export HIP_ROOT_DIR=${HIP_ROOT_DIR:-/opt/rocm/core-7.14} -export PYTHONPATH="$T${PYTHONPATH:+:$PYTHONPATH}" -export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 -export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-4,5,6,7} -export VLLM_WORKER_MULTIPROC_METHOD=spawn -export GPU_MAX_HW_QUEUES=2 -export VLLM_ROCM_USE_AITER=0 VLLM_ROCM_USE_AITER_MOE=0 -export VLLM_RDNA_FORCE_FP16=1 TORCH_BLAS_PREFER_HIPBLASLT=0 VLLM_BATCH_INVARIANT=0 -export VLLM_CACHE_ROOT=${VLLM_CACHE_ROOT:-$T/cache/step0-$TAG} -export TRITON_CACHE_DIR=$VLLM_CACHE_ROOT/triton -export TORCHINDUCTOR_CACHE_DIR=$VLLM_CACHE_ROOT/inductor -export TORCH_EXTENSIONS_DIR=$VLLM_CACHE_ROOT/extensions -export VLLM_USE_RDNA2_FA=1 VLLM_FA_RDNA2_GQA_DECODE=1 -export VLLM_USE_V2_MODEL_RUNNER=1 -export VLLM_ROCM_MOE_PREFILL=0 VLLM_GDN_HIP_PREFILL=0 -export VLLM_RDNA_FUSED_SE=1 -export VLLM_RDNA_DENSE_INT8=0 VLLM_RDNA_DENSE_INT8_ONLY=0 VLLM_RDNA_DENSE_GEMV=0 -export VLLM_RDNA_AR=1 VLLM_RDNA_AR_MAX_KB=64 VLLM_RDNA_AR_ONESHOT_KB=64 -export VLLM_RDNA_AR_BLOCKS=0 VLLM_RDNA_AR_PACE=0 -export VLLM_CAUSAL_CONV1D_RDNA2_FWD=${CONV1D:-0} -export VLLM_CAUSAL_CONV1D_RDNA2_UPDATE=${CONV1D:-0} -export VLLM_ENABLE_STARTUP_PLAN=0 -export VLLM_TUNED_CONFIG_FOLDER=$T/tuned-moe -export VLLM_PLE_CPU_OFFLOAD=1 VLLM_PLE_QUANT_DIR=$PLE -export HSA_FORCE_FINE_GRAIN_PCIE=1 HSA_ENABLE_SDMA=0 -export OMP_NUM_THREADS=4 TOKENIZERS_PARALLELISM=false PYTHONFAULTHANDLER=1 - -# TunableOp: shared build-keyed rows (keeps GEMM selection deterministic). -# shellcheck source=tools/rdna2_028/tunableop_env.sh -source "$T/tools/rdna2_028/tunableop_env.sh" -configure_tunableop "$RSL/librocblas.so.5" "$T/tunableop" 2>&1 | tee -a "$DRIVER_LOG" - -# --- optional .so swap (for A/B; default = whatever is installed) ----------- -if [[ -n $READY_SO ]]; then - cp "$READY_SO" "$T/vllm/_rocm_C.abi3.so" -fi -so=$(sha256sum "$T/vllm/_rocm_C.abi3.so" | cut -c1-16) - -# --- PCI-SERR guard --------------------------------------------------------- -sel_count(){ timeout 10 sudo -n ipmitool sel list 2>/dev/null | grep -c 'PCI SERR'; } -ss0=$(sel_count) -log "=== STEP0 MTP=$MTP tag=$TAG so=$so cells='$CELLS' ===" -log "PCI-SERR before=$ss0 uptime=$(uptime -p)" - -# --- model compilation config ---------------------------------------------- -COMPILE='{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,4,8],"compile_ranges_endpoints":[]}' -SPEC=() -if [[ $MTP != 0 ]]; then - SPEC=(--speculative-config "{\"method\":\"mtp\",\"num_speculative_tokens\":$MTP,\"use_local_argmax_reduction\":true}") -fi - -common_args=(--model "$MODEL" --tensor-parallel-size 4 --dtype float16 - --block-size 1024 --max-model-len 262144 --max-num-seqs 8 - --max-num-batched-tokens 2048 --long-prefill-token-threshold 0 - --prefill-schedule-interval 1 --kv-cache-memory-bytes 4026531840 - --compilation-config "$COMPILE" --enable-expert-parallel - --enable-prefix-caching --mamba-cache-mode align - --limit-mm-per-prompt '{"image":1}' --mm-processor-kwargs '{"max_pixels":1605632}' - --attention-backend RDNA_ATTN --gpu-memory-utilization 0.90 - --dataset-name random --num-prompts "$NUM_PROMPTS" --output-len "$OUTLEN" - --seed 12345) -[[ ${#SPEC[@]} -gt 0 ]] && common_args+=("${SPEC[@]}") - -for IN in $CELLS; do - CD=$OUT/in$IN - mkdir -p "$CD" "$CD/tmp" - log "cell in=$IN out=$OUTLEN starting" - t0=$(date +%s) - TMPDIR="$CD/tmp" "$V/bin/rocprofv3" --kernel-trace -f csv -o "$CD/prof" -- \ - "$V/bin/python" -m vllm.entrypoints.cli.main bench throughput \ - "${common_args[@]}" --input-len "$IN" \ - >"$CD/bench.log" 2>&1 - rc=$? - t1=$(date +%s) - log "cell in=$IN rc=$rc elapsed=$((t1-t0))s" - grep -m1 -iE "Throughput|Output token throughput|Mean TTFT|Mean TPOT" "$CD/bench.log" | tr -s ' ' | tee -a "$DRIVER_LOG" - ls -la "$CD"/prof*.csv 2>/dev/null | tee -a "$DRIVER_LOG" || log "NO CSV for in=$IN" - ss=$(sel_count) - if [[ "$ss" -gt "$ss0" ]]; then - log "STOP NEW PCI SERR ($ss0 -> $ss)"; echo "STOP NEW PCI SERR" >"$OUT/IM_STOP"; break - fi -done -log "=== STEP0 done MTP=$MTP ===" diff --git a/tools/rdna2_028/step0_torchprof.sh b/tools/rdna2_028/step0_torchprof.sh deleted file mode 100644 index e5e32a7d02d6..000000000000 --- a/tools/rdna2_028/step0_torchprof.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# STEP-0 in-model kernel profile via vLLM's built-in torch profiler. -# -# Why not rocprofv3: its spawn-mode launcher mishandles vLLM's -# multiprocessing-spawn process tree (the wrapped engine is reported as a -# 0.000 s child, the per-worker rocprofiler buffers are never merged, and no -# CSV is produced). vLLM's own profiler runs inside each worker and writes a -# complete chrome trace on stop_profile(), so it survives the spawn tree. -# -# Kernels are profiled with --enforce-eager so graph replay does not collapse -# them into a hipGraphLaunch. The kernel set (MTP sampling trio, QSA, conv1d, -# GEMMs) is the same as the captured path; shares are a ranking, not a -# per-phase budget. -# -# MTP=0 TAG=step0_tp_mtp0 bash tools/rdna2_028/step0_torchprof.sh -# -# No /tmp. Artifacts: /home/chenco_adm/w4a8_runs//. -set -uo pipefail - -V=${V:-/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0} -T=${T:-/home/chenco_adm/vllm-rdna-0.28.0} -MODEL=${MODEL:-/home/chenco_adm/hfcache/hub/models--wtdcode--Qwen3.8-Flash-Next-AWQ-W4A16/snapshots/0939125b929543a783ce700c90e36dd1a575c00c} -PLE=${PLE:-/home/chenco_adm/hfcache/hub/models--primitive-ai--Qwen3.8-Flash-Next-PLE-quant/snapshots/4f861b63f69e61bfc2e22130ec91ec67f03ec43e/ples_int4} -MTP=${MTP:-0} -OUTLEN=${OUTLEN:-128} -TAG=${TAG:-step0_tp_mtp$MTP} -OUT=${OUT:-/home/chenco_adm/w4a8_runs/$TAG} -CELLS=${CELLS:-"16384 1024"} -READY_SO=${READY_SO:-} - -mkdir -p "$OUT" -DRIVER_LOG=$OUT/driver.log -log(){ echo "[$(date +%H:%M:%S)] $*" | tee -a "$DRIVER_LOG"; } - -RSL=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -RS=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$RSL:$RS/host-math/lib:$RS/rocm_sysdeps/lib:$RS/core/lib:$V/lib/python3.12/site-packages/torch/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -export ROCM_HOME=${ROCM_HOME:-/opt/rocm/core-7.14} ROCM_PATH=${ROCM_PATH:-/opt/rocm/core-7.14} -export PYTHONPATH="$T${PYTHONPATH:+:$PYTHONPATH}" -export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 -export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-4,5,6,7} -export VLLM_WORKER_MULTIPROC_METHOD=spawn GPU_MAX_HW_QUEUES=2 -export VLLM_ROCM_USE_AITER=0 VLLM_ROCM_USE_AITER_MOE=0 -export VLLM_RDNA_FORCE_FP16=1 TORCH_BLAS_PREFER_HIPBLASLT=0 VLLM_BATCH_INVARIANT=0 -export VLLM_CACHE_ROOT=${VLLM_CACHE_ROOT:-$T/cache/step0tp-$TAG} -export TRITON_CACHE_DIR=$VLLM_CACHE_ROOT/triton TORCHINDUCTOR_CACHE_DIR=$VLLM_CACHE_ROOT/inductor TORCH_EXTENSIONS_DIR=$VLLM_CACHE_ROOT/extensions -export VLLM_USE_RDNA2_FA=1 VLLM_FA_RDNA2_GQA_DECODE=1 VLLM_USE_V2_MODEL_RUNNER=1 -export VLLM_ROCM_MOE_PREFILL=0 VLLM_GDN_HIP_PREFILL=0 VLLM_RDNA_FUSED_SE=1 -export VLLM_RDNA_DENSE_INT8=0 VLLM_RDNA_DENSE_INT8_ONLY=0 VLLM_RDNA_DENSE_GEMV=0 -export VLLM_RDNA_AR=1 VLLM_RDNA_AR_MAX_KB=64 VLLM_RDNA_AR_ONESHOT_KB=64 VLLM_RDNA_AR_BLOCKS=0 VLLM_RDNA_AR_PACE=0 -export VLLM_CAUSAL_CONV1D_RDNA2_FWD=${CONV1D:-0} VLLM_CAUSAL_CONV1D_RDNA2_UPDATE=${CONV1D:-0} -export VLLM_ENABLE_STARTUP_PLAN=0 VLLM_TUNED_CONFIG_FOLDER=$T/tuned-moe -export VLLM_PLE_CPU_OFFLOAD=1 VLLM_PLE_QUANT_DIR=$PLE -export HSA_FORCE_FINE_GRAIN_PCIE=1 HSA_ENABLE_SDMA=0 OMP_NUM_THREADS=4 TOKENIZERS_PARALLELISM=false PYTHONFAULTHANDLER=1 -source "$T/tools/rdna2_028/tunableop_env.sh" -configure_tunableop "$RSL/librocblas.so.5" "$T/tunableop" 2>&1 | tee -a "$DRIVER_LOG" - -if [[ -n $READY_SO ]]; then cp "$READY_SO" "$T/vllm/_rocm_C.abi3.so"; fi -so=$(sha256sum "$T/vllm/_rocm_C.abi3.so" | cut -c1-16) -sel_count(){ timeout 10 sudo -n ipmitool sel list 2>/dev/null | grep -c 'PCI SERR'; } -ss0=$(sel_count) -log "=== STEP0-TP MTP=$MTP tag=$TAG so=$so conv1d=${CONV1D:-0} cells='$CELLS' (eager) ===" -log "PCI-SERR before=$ss0 uptime=$(uptime -p)" - -SPEC=() -[[ $MTP != 0 ]] && SPEC=(--speculative-config "{\"method\":\"mtp\",\"num_speculative_tokens\":$MTP,\"use_local_argmax_reduction\":true}") - -for IN in $CELLS; do - CD=$OUT/in$IN - rm -rf "$CD"; mkdir -p "$CD/prof" - PROFCFG="{\"profiler\":\"torch\",\"torch_profiler_dir\":\"$CD/prof\",\"torch_profiler_use_gzip\":false,\"torch_profiler_with_stack\":false,\"torch_profiler_record_shapes\":false,\"torch_profiler_with_memory\":false,\"torch_profiler_dump_cuda_time_total\":true}" - log "cell in=$IN out=$OUTLEN starting" - t0=$(date +%s) - "$V/bin/python" -m vllm.entrypoints.cli.main bench throughput \ - --model "$MODEL" --tensor-parallel-size 4 --dtype float16 \ - --block-size 1024 --max-model-len 262144 --max-num-seqs 8 \ - --max-num-batched-tokens 2048 --long-prefill-token-threshold 0 \ - --prefill-schedule-interval 1 --kv-cache-memory-bytes 4026531840 \ - --enable-expert-parallel --enable-prefix-caching --mamba-cache-mode align \ - --limit-mm-per-prompt '{"image":1}' --mm-processor-kwargs '{"max_pixels":1605632}' \ - --attention-backend RDNA_ATTN --gpu-memory-utilization 0.90 \ - --dataset-name random --num-prompts 1 --output-len "$OUTLEN" --seed 12345 \ - --input-len "$IN" --enforce-eager --profile --profiler-config "$PROFCFG" \ - "${SPEC[@]}" >"$CD/bench.log" 2>&1 - rc=$? - log "cell in=$IN rc=$rc elapsed=$(( $(date +%s) - t0 ))s" - ls -la "$CD/prof"/ 2>/dev/null | tail -6 | tee -a "$DRIVER_LOG" - ss=$(sel_count); if [[ "$ss" -gt "$ss0" ]]; then log "STOP NEW PCI SERR"; echo STOP >"$OUT/IM_STOP"; break; fi -done -log "=== STEP0-TP done MTP=$MTP ===" diff --git a/tools/rdna2_028/step0_tp_parse.py b/tools/rdna2_028/step0_tp_parse.py deleted file mode 100644 index dd8107dbac4d..000000000000 --- a/tools/rdna2_028/step0_tp_parse.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Aggregate vLLM torch-profiler traces into a kernel-share table. - -usage: step0_tp_parse.py [--json out.json] - -Each TP worker writes its own trace; the table sums device kernel time across -all workers (a kernel's absolute time is wall-clock across concurrent ranks, so -this is per-rank-busy summed, i.e. a ranking of where GPU time goes). -""" -from __future__ import annotations - -import glob -import json -import os -import re -import sys -from collections import defaultdict - -GROUPS = [ - ("mtp_stats", re.compile(r"compute_local_logits_stats", re.I)), - ("mtp_reject", re.compile(r"rejection_kernel", re.I)), - ("mtp_resample", re.compile(r"resample_kernel", re.I)), - ("qsa", re.compile(r"qsa|sparse_paged_gqa|expand_qsa|compress_qsa", re.I)), - ("causal_conv1d_triton", re.compile(r"causal_conv1d_(fwd|update)_kernel", re.I)), - ("causal_conv1d_hip", re.compile(r"causal_conv1d_(fwd|update)_rdna2", re.I)), - ("fa_rdna2", re.compile(r"fa_rdna2", re.I)), - ("paged_mqa_indexer", re.compile(r"paged_mqa_logits", re.I)), - ("rope", re.compile(r"rope|rotary", re.I)), - ("triton", re.compile(r"triton_", re.I)), - ("rocblas", re.compile(r"rocblas|Cijk_|gemm_|Gemm|GEMM", re.I)), - ("w4a16_dot2", re.compile(r"gptq_gemm_rdna2|moe_gptq_gemm_rdna2|w4a8|dot2", re.I)), - ("allreduce", re.compile(r"all_reduce|allreduce|rdna_ar|nccl", re.I)), -] - - -def group_of(name: str) -> str: - for g, rx in GROUPS: - if rx.search(name): - return g - return "other" - - -def main() -> int: - d = sys.argv[1] - files = sorted(glob.glob(os.path.join(d, "**", "*.json"), recursive=True)) - per_kernel: dict[str, list[int]] = defaultdict(lambda: [0, 0]) - per_group: dict[str, list[int]] = defaultdict(lambda: [0, 0]) - total = 0 - nfiles = 0 - for f in files: - try: - obj = json.load(open(f)) - except Exception: - continue - evs = obj.get("traceEvents", obj if isinstance(obj, list) else []) - if not evs: - continue - nfiles += 1 - for e in evs: - if not isinstance(e, dict): - continue - cat = str(e.get("cat", "")) - if cat.lower() not in ("kernel", "gpu_memcpy", "gpu_memset"): - continue - name = e.get("name", "") or "" - dur = e.get("dur") - if not isinstance(dur, (int, float)) or dur < 0: - continue - per_kernel[name][0] += int(dur) - per_kernel[name][1] += 1 - per_group[group_of(name)][0] += int(dur) - per_group[group_of(name)][1] += 1 - total += int(dur) - - def fmt(d_): - return f"{d_/1000:,.1f}" - - print(f"# traces={nfiles} files_matched={len(files)} total_kernel_busy_ms={fmt(total)}") - print() - print("## by group") - print(f"{'group':26} {'busy_ms':>13} {'share':>7} {'kernels':>10}") - for g, (dd, n) in sorted(per_group.items(), key=lambda kv: -kv[1][0]): - print(f"{g:26} {fmt(dd):>13} {100*dd/total:6.1f}% {n:>10}") - print() - print("## top kernels") - print(f"{'busy_ms':>13} {'share':>7} {'n':>9} kernel") - for name, (dd, n) in sorted(per_kernel.items(), key=lambda kv: -kv[1][0])[:45]: - short = name if len(name) <= 110 else name[:107] + "..." - print(f"{fmt(dd):>13} {100*dd/total:6.1f}% {n:>9} {short}") - - if "--json" in sys.argv: - out = sys.argv[sys.argv.index("--json") + 1] - json.dump( - { - "total_ms": total / 1000, - "groups": {g: {"ms": dd / 1000, "n": n} for g, (dd, n) in per_group.items()}, - "kernels": {k: {"ms": dd / 1000, "n": n} for k, (dd, n) in per_kernel.items()}, - }, - open(out, "w"), - indent=2, - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/rdna2_028/tunableop_rows_e2e.sh b/tools/rdna2_028/tunableop_rows_e2e.sh deleted file mode 100644 index c0fd61459ac3..000000000000 --- a/tools/rdna2_028/tunableop_rows_e2e.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# One production Flash-Next arm (W4A16 or W4A8, MTP=0) at the four cells with -# whatever repo rows are currently installed. Thin wrapper over -# flashnext_w4a8_arm.sh that records the box state around the run. -# -# ARM=w4a8off W4A8=0 TAG=post-w4a16 PORT=18270 bash tunableop_rows_e2e.sh -set -uo pipefail - -T=${VLLM_TREE:-/home/chenco_adm/vllm-rdna-0.28.0} -W=${WORK:-/home/chenco_adm/w4a8_runs/tunableop-rows} -ARM=${ARM:?set ARM (w4a8on|w4a8off)} -W4A8=${W4A8:?set W4A8 (0|1)} -TAG=${TAG:?set TAG} -PORT=${PORT:-18270} -OUT=${OUT:-$W/e2e} -CELLS=${CELLS:-"1 1024 512 221|8 1024 512 222|1 16384 1024 111|8 16384 1024 112"} -mkdir -p "$OUT" - -sel() { sudo -n ipmitool sel list 2>/dev/null | grep -c 'PCI SERR'; } -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$OUT/$TAG.wrapper.log"; } - -log "=== e2e $TAG ARM=$ARM W4A8=$W4A8 PORT=$PORT uptime=$(uptime -p) PCI-SERR=$(sel) ===" - -ARM=$ARM W4A8=$W4A8 MTP=0 PORT=$PORT OUT=$OUT TAG=$TAG CELLS=$CELLS \ - bash "$T/tools/rdna2_028/flashnext_w4a8_arm.sh" - -log "--- $TAG cells.csv ---" -cat "$OUT/$TAG/cells.csv" 2>/dev/null | tee -a "$OUT/$TAG.wrapper.log" -log "--- $TAG markers (W4A8 arm) ---" -grep -E "W4A8 sdot4 path active|rdna_ar:" "$OUT/$TAG/markers.txt" 2>/dev/null | head -8 | tee -a "$OUT/$TAG.wrapper.log" -log "=== e2e $TAG done status=$(cat "$OUT/$TAG/status.txt" 2>/dev/null) PCI-SERR=$(sel) uptime=$(uptime -p) ===" diff --git a/tools/rdna2_028/w4a8_ab.sh b/tools/rdna2_028/w4a8_ab.sh deleted file mode 100644 index 3fa99eadbb94..000000000000 --- a/tools/rdna2_028/w4a8_ab.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# Controlled W4A8 A/B driver for the 27B dense MTP stack. One arm per run: -# MTP=2 W4A8=1 TAG=ab_m2_w1 bash w4a8_ab.sh -# MTP=2 W4A8=0 TAG=ab_m2_w0 bash w4a8_ab.sh -# -# Starts the server via tools/rdna/serve_gfx1030_27b_dense.sh with the env-gated -# shape diagnostic (VLLM_RDNA2_W4A8_DEBUG=1, csrc/rocm/w4a8_sdot4_rdna2.cu), -# WARMS UP with throwaway requests, runs greedy coherence probes, then the -# production bench cells with --temperature=0. Extracts: -# - every per-10s SpecDecoding window (Mean acceptance length + acc/draft), -# - the distinct (m,k,n,group) shapes that fired the W4A8 fast path, -# - the W4A8 gate marker. -# Everything lands in /home/chenco_adm/w4a8_runs// (persistent, no /tmp). -set -uo pipefail -V=/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0 -T=/home/chenco_adm/vllm-rdna-0.28.0 -MODEL=${MODEL:-/home/chenco_adm/.cache/huggingface/hub/models--cyankiwi--Qwen3.8-27B-AWQ-INT4/snapshots/63768c10df38c0395e12ef49edac1bd539eaeeea} -MTP=${MTP:-2} -W4A8=${W4A8:-0} -DBG=${DBG:-1} -TAG=${TAG:-ab_mtp${MTP}_w${W4A8}} -PORT=${PORT:-18230} -OUT=${OUT:-/home/chenco_adm/w4a8_runs} -D=$OUT/$TAG -rm -rf "$D"; mkdir -p "$D" -ROCM_SDK_LIB=$V/lib/python3.12/site-packages/_rocm_sdk_libraries/lib -ROCM_SDK=$V/lib/python3.12/site-packages/_rocm_sdk_core/lib -export LD_LIBRARY_PATH="$ROCM_SDK_LIB:$ROCM_SDK/host-math/lib:$ROCM_SDK/rocm_sysdeps/lib:$ROCM_SDK/core/lib:$V/lib/python3.12/site-packages/torch/lib" - -log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$D/driver.log"; } - -# Cells: "n_prompts input_len output_len seed". 16k c=8 last (slowest, and the -# load under which the MTP acceptance collapse was observed). -CELLS=${CELLS:-"1 16384 1024 111|1 1024 512 221|8 1024 512 222|8 16384 1024 112"} - -setsid nohup env MTP=$MTP W4A8=$W4A8 VLLM_RDNA2_W4A8_DEBUG=$DBG EAGER=${EAGER:-0} ATTN=fa TP=${TP:-4} PORT=$PORT \ - KV=${KV:-8000000000} SEQS=${SEQS:-8} MAXBAT=${MAXBAT:-2048} \ - CG_MODE=${CG_MODE:-FULL_AND_PIECEWISE} MODEL="$MODEL" \ - bash "$T/tools/rdna/serve_gfx1030_27b_dense.sh" > "$D/serve.log" 2>&1 < /dev/null & -log "serve pid $! (MTP=$MTP W4A8=$W4A8 DBG=$DBG)" -ready=0 -for i in $(seq 1 96); do - sleep 10 - curl -s --max-time 4 "http://127.0.0.1:$PORT/v1/models" 2>/dev/null | grep -q q27d && { ready=1; log "READY t=$((i*10))s"; break; } -done -[ "$ready" = "1" ] || { log "NOT READY"; tail -40 "$D/serve.log"; exit 1; } - -# --- Warmup (throwaway; cold-start pollution is real on this stack) --- -log "warmup start" -for k in 1 2; do - curl -s --max-time 300 -X POST "http://127.0.0.1:$PORT/v1/completions" -H 'Content-Type: application/json' \ - -d "{\"model\":\"q27d\",\"prompt\":\"$(printf 'word %.0s' $(seq 1 2000))\",\"max_tokens\":8,\"temperature\":0,\"ignore_eos\":true}" >/dev/null -done -log "warmup done" - -# --- Coherence probe (greedy) --- -log "coherence probe start" -"$V/bin/python" "$T/tools/rdna2_028/probe_w4a8.py" "http://127.0.0.1:$PORT/v1" q27d 1 | tee "$D/coherence.txt" -log "coherence probe done" - -# Run the bench client from the output dir (no vllm/ package shadowing, and no -# /tmp on this box). -cd "$D" -IFS='|' read -ra _cells <<< "$CELLS" -for cell in "${_cells[@]}"; do - set -- $cell - log "bench c=$1 in=$2 out=$3 start" - "$V/bin/python" -m vllm.entrypoints.cli.main bench serve \ - --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" \ - --model "$MODEL" --served-model-name q27d --dataset-name random \ - --random-input-len "$2" --random-output-len "$3" --num-prompts "$1" --max-concurrency "$1" \ - --ignore-eos --request-rate inf --seed "$4" --temperature 0 \ - --save-result --result-dir "$D/c$2_$1" > "$D/c$2_$1.log" 2>&1 - log "bench c=$1 in=$2 out=$3 done: $(grep -m1 'Output token throughput' "$D/c$2_$1.log" | tr -s ' ')" -done - -# --- Extract --- -log "W4A8 gate: $(grep -m1 'W4A8 sdot4 path active' "$D/serve.log" || echo '(not taken)')" -{ - echo "=== W4A8 fast-path shapes (unique, all ranks) ===" - grep -h "W4A8-DEBUG" "$D/serve.log" | sort -u -} | tee "$D/shapes.txt" -{ - echo "=== SpecDecoding windows (serve log) ===" - grep "SpecDecoding metrics" "$D/serve.log" \ - | sed -E "s/^\(APIServer[^)]*\) INFO ([0-9-]+ [0-9:]+).*Mean acceptance length: ([0-9.]+).*Accepted: ([0-9]+) tokens, Drafted: ([0-9]+).*/\1 mean=\2 acc=\3 dr=\4/" -} | tee "$D/acceptance.txt" - -# Teardown scoped to this tree's cache root. -api=$(grep -oE "APIServer pid=[0-9]+" "$D/serve.log" | head -1 | cut -d= -f2) -[ -n "$api" ] && kill -TERM "$api" 2>/dev/null && sleep 10 -for p in $(pgrep -f "VLLM::Worker|VLLM::EngineCore|entrypoints.cli.main serve" 2>/dev/null); do tr '\0' '\n' < /proc/$p/environ 2>/dev/null | grep -q "VLLM_CACHE_ROOT=$T" && kill -9 $p 2>/dev/null; done -log "done $TAG" diff --git a/tools/rdna2_028/wt_ab.sh b/tools/rdna2_028/wt_ab.sh deleted file mode 100644 index 0bb8287287c5..000000000000 --- a/tools/rdna2_028/wt_ab.sh +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# -# Generic in-model A/B driver over tools/rdna/serve_rdna.sh RECIPE=... -# -# One arm per invocation: boot the recipe, warm, coherence probe, run cells, -# tear down. Swap only the thing under test (VLLM_CAUSAL_CONV1D_RDNA2_*, -# ATTN, or the installed _rocm_C .so) between arms; keep the compile cache -# shared so the graph is otherwise byte-identical. -# -# RECIPE=flashnext-mtp0 SERVED=flash-next ARM=triton CONV1D=0 \ -# CACHE=$T/cache/wt-ab-fn PORT=18150 TAG=item2_triton \ -# CELLS="1 16384 512 711|1 1024 512 721" bash tools/rdna2_028/wt_ab.sh -# -# Env knobs: -# RECIPE SERVED MODEL PORT ARM TAG CACHE KEEP_CACHE CELLS WARMUP READY_MAX -# SO optional .so to install into vllm/_rocm_C.abi3.so -# CONV1D optional 0|1 -> both VLLM_CAUSAL_CONV1D_RDNA2_{FWD,UPDATE} -# ATTN optional fa|triton|none -# EXTRA_ARGS extra trailing KEY=value overrides for serve_rdna.sh -# PLE optional VLLM_PLE_QUANT_DIR (only set when non-empty) -# HIP_VISIBLE GPU list (default 4,5,6,7) -# -# No /tmp. Logs: /home/chenco_adm/w4a8_runs//. -set -uo pipefail - -V=${V:-/home/chenco_adm/Apps/vllm/venv-7.14.0_0.28.0} -T=${T:-/home/chenco_adm/vllm-rdna-0.28.0} -RUN=${RUN:-/home/chenco_adm/w4a8_runs} -RECIPE=${RECIPE:?RECIPE required} -SERVED=${SERVED:-} -ARM=${ARM:-arm} -PORT=${PORT:-18150} -TAG=${TAG:-wt_${RECIPE}_${ARM}} -CELLS=${CELLS:-"1 16384 512 711|1 1024 512 721"} -WARMUP=${WARMUP:-"1024 64 991|16384 64 992"} -READY_MAX=${READY_MAX:-180} -KEEP_CACHE=${KEEP_CACHE:-0} -HIP_VISIBLE=${HIP_VISIBLE:-4,5,6,7} -CACHE=${CACHE:-$T/cache/wt-ab-$RECIPE} - -D=$RUN/$TAG -mkdir -p "$D/cells" -DRIVER_LOG=$D/driver.log -log(){ echo "[$(date +%H:%M:%S)] $*" | tee -a "$DRIVER_LOG"; } -sel_count(){ timeout 10 sudo -n ipmitool sel list 2>/dev/null | grep -c 'PCI SERR'; } - -our_pids() { - local p e - for p in $(pgrep -f "entrypoints.cli.main serve|entrypoints.openai.api_server|VLLM::Worker|VLLM::EngineCore|PleOffloadWorker" 2>/dev/null); do - e=$(tr '\0' '\n' <"/proc/$p/environ" 2>/dev/null || true) - case "$e" in *"VLLM_CACHE_ROOT=$CACHE"*) echo "$p" ;; esac - done -} -hygiene() { - local n=0 left p i - for p in $(our_pids); do kill -TERM "$p" 2>/dev/null && n=$((n+1)); done - [ "$n" -gt 0 ] && sleep 8 - for p in $(our_pids); do kill -KILL "$p" 2>/dev/null; done - for i in $(seq 1 30); do left=$(our_pids | tr '\n' ' '); [ -z "${left// /}" ] && break; sleep 2; done - left=$(our_pids | tr '\n' ' ') - log "hygiene: terminated=$n remaining=[${left% }]" - [ -z "${left// /}" ] -} - -if [[ -n ${SO:-} ]]; then - [ -r "$SO" ] || { log "missing SO=$SO"; exit 1; } - cp "$SO" "$T/vllm/_rocm_C.abi3.so" -fi -so=$(sha256sum "$T/vllm/_rocm_C.abi3.so" | cut -c1-16) -log "=== ARM=$ARM recipe=$RECIPE served=$SERVED port=$PORT so=$so conv1d=${CONV1D:-} attn=${ATTN:-} ===" -hygiene || { log "hygiene failed"; exit 1; } -if ss -ltn 2>/dev/null | grep -q ":$PORT "; then log "port $PORT busy"; exit 1; fi - -if [ "$KEEP_CACHE" = 1 ] && [ -d "$CACHE/inductor" ]; then - log "KEEP_CACHE=1: reusing $CACHE" -else - rm -rf "$CACHE" -fi -mkdir -p "$CACHE/inductor" "$CACHE/extensions" "$CACHE/triton" "$CACHE/vllm" -rm -f "$CACHE/rdna_ar_wedged" - -# Trailing KEY=value overrides passed through to serve_rdna.sh. -declare -a ov=() -[[ -n ${SERVED:-} ]] && ov+=(SERVED_NAME="$SERVED") -[[ -n ${CONV1D:-} ]] && ov+=(VLLM_CAUSAL_CONV1D_RDNA2_FWD="$CONV1D" VLLM_CAUSAL_CONV1D_RDNA2_UPDATE="$CONV1D") -[[ -n ${ATTN:-} ]] && ov+=(ATTN="$ATTN") -# shellcheck disable=SC2206 -[[ -n ${EXTRA_ARGS:-} ]] && ov+=($EXTRA_ARGS) - -S0=$(sel_count); log "PCI-SERR before=$S0 uptime=$(uptime -p)" -T0=$(date +%s) -setsid nohup env \ - VENV="$V" VLLM_TREE="$T" MODEL="${MODEL:-}" ${PLE:+VLLM_PLE_QUANT_DIR="$PLE"} \ - VLLM_CACHE_ROOT="$CACHE" TORCHINDUCTOR_CACHE_DIR="$CACHE/inductor" \ - TRITON_CACHE_DIR="$CACHE/triton" TORCH_EXTENSIONS_DIR="$CACHE/extensions" \ - HIP_VISIBLE_DEVICES="$HIP_VISIBLE" \ - bash "$T/tools/rdna/serve_rdna.sh" RECIPE="$RECIPE" PORT="$PORT" "${ov[@]}" \ - >"$D/serve.log" 2>&1 /dev/null) - if [ -n "$models" ]; then - if [ -n "$SERVED" ]; then echo "$models" | grep -q "$SERVED" && { ready=1; break; } - else ready=1; break; fi - fi - if [ -n "$(our_pids)" ]; then dead=0; else dead=$((dead+1)); fi - [ "$dead" -ge 3 ] && { log "server gone during boot"; break; } -done -COLD=$(( $(date +%s) - T0 )) -if [ "$ready" != 1 ]; then log "NOT READY after ${COLD}s"; tail -50 "$D/serve.log" >>"$DRIVER_LOG"; hygiene; exit 1; fi -log "READY t=${COLD}s"; echo "$COLD" >"$D/cold_seconds.txt" -S1=$(sel_count); log "PCI-SERR after boot=$S1 uptime=$(uptime -p)" -if [ "$S1" -gt "$S0" ]; then echo "STOP NEW PCI SERR" >"$RUN/AB_STOP"; hygiene; exit 2; fi - -# attention backend actually in use -grep -iE "attention backend|pinning the attention backend|Using .* backend" "$D/serve.log" | tail -3 >>"$DRIVER_LOG" || true - -if [ "${BOOT_ONLY:-0}" = 1 ]; then log "BOOT_ONLY ready"; exit 0; fi - -if [ -z "$SERVED" ]; then - SERVED=$(grep -m1 -oE '"id":"[^"]+"' "$D/serve.log" 2>/dev/null | head -1 | cut -d'"' -f4 || true) -fi - -for wl in $WARMUP; do set -- $wl - ( cd "$D" && "$V/bin/python" -m vllm.entrypoints.cli.main bench serve --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" --model "${MODEL:-$SERVED}" --served-model-name "$SERVED" --dataset-name random --random-input-len "$1" --random-output-len "$2" --num-prompts 1 --max-concurrency 1 --ignore-eos --request-rate inf --seed "$3" --temperature 0 ) >"$D/warmup_$1.log" 2>&1 -done -log "warmup done" - -"$V/bin/python" - "$PORT" "$SERVED" >"$D/coherence.txt" 2>&1 <<'PY' -import json,sys,urllib.request,re -port,model=sys.argv[1],sys.argv[2] -def ask(p,mt=24): - body=json.dumps({"model":model,"prompt":p,"max_tokens":mt,"temperature":0.0,"ignore_eos":True}).encode() - r=urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions",data=body,headers={"Content-Type":"application/json"}) - with urllib.request.urlopen(r,timeout=900) as resp: return json.load(resp)["choices"][0]["text"] or "" -def garbage(t): - if not t.strip(): return "empty" - if re.search(r'(.)\1{29,}', t): return "repeat>=30" - if sum(1 for c in t if not c.isprintable() and c not in "\n\t") > len(t)*0.1: return "nonprintable" - if t.count("!")>len(t)*0.5: return "bang-flood" - return "ok" -ok=0; tot=0 -for name,p,exp in [("france","The capital of France is","paris"),("math","2 + 2 =","4")]: - tot+=1 - try: t=ask(p) - except Exception as e: t=f"" - g=garbage(t); good=(exp in t.lower()) and g=="ok"; ok+=good - print(f"[{name}] good={good} garbage={g} exp={exp!r} | {t[:100]!r}") -print(f"COHERENCE {ok}/{tot}") -PY -log "coherence: $(grep -m1 COHERENCE "$D/coherence.txt" 2>/dev/null)" - -IFS='|' read -ra _cells <<<"$CELLS" -for cell in "${_cells[@]}"; do set -- $cell - N=$1; IN=$2; OUTL=$3; SEED=$4 - CD="$D/cells/c${N}_${IN}_s${SEED}" - mkdir -p "$CD" - ( cd "$D" && "$V/bin/python" -m vllm.entrypoints.cli.main bench serve --backend openai --endpoint /v1/completions --base-url "http://127.0.0.1:$PORT" --model "${MODEL:-$SERVED}" --served-model-name "$SERVED" --dataset-name random --random-input-len "$IN" --random-output-len "$OUTL" --num-prompts "$N" --max-concurrency "$N" --ignore-eos --request-rate inf --seed "$SEED" --temperature 0 --save-result --result-dir "$CD" ) >"$CD.log" 2>&1 - log "cell c=$N in=$IN s=$SEED | $(grep -m1 'Output token throughput' "$CD.log" | tr -s ' ') | $(grep -m1 'Mean TPOT' "$CD.log" | tr -s ' ')" - S2=$(sel_count); if [ "$S2" -gt "$S0" ]; then echo "STOP NEW PCI SERR" >"$RUN/AB_STOP"; break; fi -done - -{ echo "cell,arm,seed,requests,output_tok_s,total_tok_s,mean_ttft_ms,mean_tpot_ms,median_itl_ms,p99_itl_ms" - for cell in "${_cells[@]}"; do set -- $cell - N=$1; IN=$2; SEED=$4 - CD="$D/cells/c${N}_${IN}_s${SEED}" - f=$(ls "$CD"/*.json 2>/dev/null | head -1) - "$V/bin/python" - "$N" "$IN" "$SEED" "$ARM" "$f" <<'PY' -import json,sys -n,inn,seed,arm,f=sys.argv[1:6] -d={} -try: d=json.load(open(f)) -except Exception: pass -def g(*ks): - for k in ks: - if k in d and d[k] not in (None,""): return d[k] - return "" -print(f"c{n}_{inn},{arm},{seed},{g('completed','successful_requests')},{g('output_throughput')},{g('total_token_throughput')},{g('mean_ttft_ms','mean_ttft')},{g('mean_tpot_ms','mean_tpot')},{g('median_itl_ms')},{g('p99_itl_ms')}") -PY - done; } >"$D/summary.csv" 2>"$D/summary.err" - -hygiene -S3=$(sel_count); log "PCI-SERR after teardown=$S3 uptime=$(uptime -p)" -log "arm $ARM done"