From 6645303fca7148b2652ac1da0e6927429d5438ff Mon Sep 17 00:00:00 2001 From: Rui Zhang Date: Mon, 13 Jul 2026 16:46:52 -0700 Subject: [PATCH 1/3] add vllm + lmcache on amd mi355x Signed-off-by: Rui Zhang --- benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh | 7 +++++-- configs/amd-master.yaml | 2 +- perf-changelog.yaml | 8 ++++++++ utils/matrix_logic/generate_sweep_configs.py | 4 ++++ utils/matrix_logic/validation.py | 2 ++ 5 files changed, 20 insertions(+), 3 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh index 241f042ff2..ce3b9b7584 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm.sh @@ -276,8 +276,7 @@ EOF git clone https://github.com/LMCache/LMCache.git cd LMCache - # https://github.com/LMCache/LMCache/pull/3853 - git checkout 9229067cec0b3a63bb8a39368d101db7ac0bc3c1 + git checkout v0.5.1 pip install -r requirements/build.txt pip install grpcio==1.78.0 CXX=hipcc BUILD_WITH_HIP=1 pip install -e . --no-build-isolation @@ -317,6 +316,9 @@ EOF LMCACHE_TX_MODE="lmcache_driven" echo "Starting LMCache MP server..." + # TEMP DEBUG: remove before merging + df -h /dev/shm + free -h LMCACHE_CMD=( lmcache server --host "$LMCACHE_HOST" @@ -330,6 +332,7 @@ EOF --max-workers "$LMCACHE_MAX_WORKERS" --eviction-policy LRU --supported-transfer-mode "$LMCACHE_TX_MODE" + --no-separate-object-group ) printf '%q ' "${LMCACHE_CMD[@]}" > "$RESULT_DIR/lmcache_command.txt" printf '\n' >> "$RESULT_DIR/lmcache_command.txt" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0f547cba86..e05b0a8069 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1422,7 +1422,7 @@ qwen3.5-fp8-mi355x-sglang-agentic-hicache: # DP-attention on/off and EP=8. dsv4-fp4-mi355x-vllm-agentic: - image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa + image: vllm/vllm-openai-rocm:v0.25.1 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 335245a6c3..defaff7ef2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5060,3 +5060,11 @@ - "Re-pin VLLM_ROUTER_IMAGE to vllm/vllm-router:nightly-20260716-1fbcde7 (previous nightly-20260629-e667ebb was garbage-collected from Docker Hub)" - "Exclude known-bad nodes mia1-p01-g09,g14 from the disagg node pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2301 + +- config-keys: + - dsv4-fp4-mi355x-vllm-agentic + description: + - "Bump LMCache source build from pinned commit 9229067 to the v0.5.1 release tag (CXX=hipcc BUILD_WITH_HIP=1)" + - "Add --no-separate-object-group to the LMCache MP server launch command" + - "Bump image from vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa (rotated off Docker Hub retention, 404s) to the stable vllm/vllm-openai-rocm:v0.25.1 release tag" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2216 diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 7df1a75b64..a3005639c4 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -783,6 +783,7 @@ def generate_full_sweep(args, all_config_data, runner_data): ), Fields.DISAGG.value: disagg, Fields.SCENARIO_TYPE.value: "agentic-coding", + Fields.RUN_EVAL.value: False, } if kv_offload_backend is not None: entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend @@ -816,6 +817,7 @@ def generate_full_sweep(args, all_config_data, runner_data): + (f"_spec-{spec_decoding}" if spec_decoding != "none" else "") ), Fields.SCENARIO_TYPE.value: "agentic-coding", + Fields.RUN_EVAL.value: False, } if kv_offload_backend is not None: entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend @@ -1081,6 +1083,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): ), Fields.DISAGG.value: disagg, Fields.SCENARIO_TYPE.value: "agentic-coding", + Fields.RUN_EVAL.value: False, } if kv_offload_backend is not None: entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend @@ -1113,6 +1116,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): + (f"_spec-{spec_decoding}" if spec_decoding != "none" else "") ), Fields.SCENARIO_TYPE.value: "agentic-coding", + Fields.RUN_EVAL.value: False, } if kv_offload_backend is not None: entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 9f19572f08..8d1779a43b 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -334,6 +334,8 @@ class MultiNodeAgenticMatrixEntry(BaseModel): exp_name: str = Field(alias=Fields.EXP_NAME.value) disagg: bool scenario_type: str = Field(alias=Fields.SCENARIO_TYPE.value) + run_eval: bool = Field(alias=Fields.RUN_EVAL.value) + eval_only: bool = Field(alias=Fields.EVAL_ONLY.value, default=False) @model_validator(mode='after') def validate_worker_hardware_pair(self): From 091006ba86ca57d69ded32a74c7c92ba787801f3 Mon Sep 17 00:00:00 2001 From: Rui Zhang Date: Thu, 23 Jul 2026 19:35:36 -0700 Subject: [PATCH 2/3] dsv4-fp4-mi355x-vllm-agentic: lmcache arm TP8->TP4, agentic evals default-on - amd-master.yaml: drop the lmcache DRAM arm from tp 8 to tp 4 (conc-list unchanged). Halves the LMCache L1 pool partition (TOTAL_CPU_DRAM_GB / (8/TP)) so the MP server finishes allocation inside the healthcheck window. - generate_sweep_configs.py: mark_eval_entries now includes agentic entries by default (include_agentic=True) - the highest-conc single-node agentic entry per (model, runner, framework, precision) group is marked run-eval in default invocations, matching --evals-only/--all-evals behavior. Co-Authored-By: Claude Fable 5 Signed-off-by: Rui Zhang --- configs/amd-master.yaml | 2 +- perf-changelog.yaml | 2 ++ utils/matrix_logic/generate_sweep_configs.py | 8 ++++---- utils/matrix_logic/test_generate_sweep_configs.py | 7 ++++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index e05b0a8069..5306b4acff 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1435,7 +1435,7 @@ dsv4-fp4-mi355x-vllm-agentic: search-space: - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 40, 48] } - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, conc-list: [64, 72], router: { name: vllm-router, version: "0.1.14" } } - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "9229067cec0b3a63bb8a39368d101db7ac0bc3c1" }, conc-list: [32, 40, 48] } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "9229067cec0b3a63bb8a39368d101db7ac0bc3c1" }, conc-list: [32, 40, 48] } # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the # amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the diff --git a/perf-changelog.yaml b/perf-changelog.yaml index defaff7ef2..47ec462758 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5067,4 +5067,6 @@ - "Bump LMCache source build from pinned commit 9229067 to the v0.5.1 release tag (CXX=hipcc BUILD_WITH_HIP=1)" - "Add --no-separate-object-group to the LMCache MP server launch command" - "Bump image from vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa (rotated off Docker Hub retention, 404s) to the stable vllm/vllm-openai-rocm:v0.25.1 release tag" + - "Drop the lmcache DRAM arm from TP8 to TP4 (conc-list [32, 40, 48] unchanged): halves the LMCache L1 pool partition (TOTAL_CPU_DRAM_GB / (8/TP)) so the MP server comes up inside the healthcheck window" + - "generate_sweep_configs.py: agentic evals now default-on in mark_eval_entries (include_agentic=True) — the highest-conc single-node agentic entry per (model, runner, framework, precision) group is marked run-eval in default test-config/full-sweep invocations, matching the --evals-only/--all-evals behavior" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2216 diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index a3005639c4..db479c5042 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -235,7 +235,7 @@ def _multinode_parallelism_key(entry: dict) -> tuple: )) -def mark_eval_entries(matrix_values: list[dict], include_agentic: bool = False) -> list[dict]: +def mark_eval_entries(matrix_values: list[dict], include_agentic: bool = True) -> list[dict]: """Eval selection policy: - Single-node: only consider 8k1k (isl=8192, osl=1024). For each unique (model, runner, framework, precision, isl, osl, spec-decoding, dp-attn): @@ -247,7 +247,8 @@ def mark_eval_entries(matrix_values: list[dict], include_agentic: bool = False) - Ignore entries with all conc values < MIN_EVAL_CONC - Mark the entry containing its highest eligible concurrency - Set eval-conc to that highest eligible concurrency - - Agentic evals are opt-in to preserve default throughput coverage. + - Agentic evals run by default: the highest-conc single-node agentic entry + per (model, runner, framework, precision) group is marked. """ from collections import defaultdict @@ -310,7 +311,6 @@ def _eligible_eval_concs(entry): eval_indices.add(best_idx) mn_eval_conc[best_idx] = best_eval_conc - # Default sweeps preserve every agentic throughput result. if include_agentic: ag_sn_groups = defaultdict(list) for i, entry in enumerate(matrix_values): @@ -1369,7 +1369,7 @@ def main(): # Apply the existing eval policy first, then expand it when requested. if not args.no_evals: - matrix_values = mark_eval_entries(matrix_values, include_agentic=args.evals_only or args.all_evals) + matrix_values = mark_eval_entries(matrix_values) if args.all_evals: matrix_values = mark_all_eval_entries(matrix_values) diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 1ad4e3c922..502a9c1429 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -233,7 +233,7 @@ def test_marks_agentic_entry_for_swebench(self): assert len(marked) == 1 assert marked[0]["conc"] == 64 - def test_default_mode_does_not_mark_agentic(self): + def test_default_mode_marks_agentic(self): matrix_values = [ { "scenario-type": "agentic-coding", @@ -250,9 +250,10 @@ def test_default_mode_does_not_mark_agentic(self): result = mark_eval_entries(matrix_values) marked = [e for e in result if e.get("run-eval")] - assert len(marked) == 0, ( - f"Expected 0 agentic entries marked run-eval in default mode, got {len(marked)}" + assert len(marked) == 1, ( + f"Expected 1 agentic entry marked run-eval in default mode, got {len(marked)}" ) + assert marked[0]["conc"] == 64 def test_single_node_skips_eval_entries_below_min_conc(self): """Single-node eval selection should ignore conc values below MIN_EVAL_CONC.""" From 9f67386a8f5f83281cb95aab33bf0db3aff42fab Mon Sep 17 00:00:00 2001 From: Rui Zhang Date: Fri, 24 Jul 2026 12:26:28 -0700 Subject: [PATCH 3/3] change to TP4 Signed-off-by: Rui Zhang --- perf-changelog.yaml | 2 +- utils/matrix_logic/generate_sweep_configs.py | 16 ++++++++-------- .../matrix_logic/test_generate_sweep_configs.py | 7 +++---- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 47ec462758..e390060c56 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5068,5 +5068,5 @@ - "Add --no-separate-object-group to the LMCache MP server launch command" - "Bump image from vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa (rotated off Docker Hub retention, 404s) to the stable vllm/vllm-openai-rocm:v0.25.1 release tag" - "Drop the lmcache DRAM arm from TP8 to TP4 (conc-list [32, 40, 48] unchanged): halves the LMCache L1 pool partition (TOTAL_CPU_DRAM_GB / (8/TP)) so the MP server comes up inside the healthcheck window" - - "generate_sweep_configs.py: agentic evals now default-on in mark_eval_entries (include_agentic=True) — the highest-conc single-node agentic entry per (model, runner, framework, precision) group is marked run-eval in default test-config/full-sweep invocations, matching the --evals-only/--all-evals behavior" + - "generate_sweep_configs.py: agentic entries now stamp run-eval: true at construction time (was false) — SWE-bench eval is ready, so agentic points default to eval-on in sweeps that skip mark_eval_entries (--no-evals); eval-marked invocations still overwrite per the existing policy" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2216 diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index db479c5042..53f842b3d7 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -235,7 +235,7 @@ def _multinode_parallelism_key(entry: dict) -> tuple: )) -def mark_eval_entries(matrix_values: list[dict], include_agentic: bool = True) -> list[dict]: +def mark_eval_entries(matrix_values: list[dict], include_agentic: bool = False) -> list[dict]: """Eval selection policy: - Single-node: only consider 8k1k (isl=8192, osl=1024). For each unique (model, runner, framework, precision, isl, osl, spec-decoding, dp-attn): @@ -247,8 +247,7 @@ def mark_eval_entries(matrix_values: list[dict], include_agentic: bool = True) - - Ignore entries with all conc values < MIN_EVAL_CONC - Mark the entry containing its highest eligible concurrency - Set eval-conc to that highest eligible concurrency - - Agentic evals run by default: the highest-conc single-node agentic entry - per (model, runner, framework, precision) group is marked. + - Agentic evals are opt-in to preserve default throughput coverage. """ from collections import defaultdict @@ -311,6 +310,7 @@ def _eligible_eval_concs(entry): eval_indices.add(best_idx) mn_eval_conc[best_idx] = best_eval_conc + # Default sweeps preserve every agentic throughput result. if include_agentic: ag_sn_groups = defaultdict(list) for i, entry in enumerate(matrix_values): @@ -783,7 +783,7 @@ def generate_full_sweep(args, all_config_data, runner_data): ), Fields.DISAGG.value: disagg, Fields.SCENARIO_TYPE.value: "agentic-coding", - Fields.RUN_EVAL.value: False, + Fields.RUN_EVAL.value: True, } if kv_offload_backend is not None: entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend @@ -817,7 +817,7 @@ def generate_full_sweep(args, all_config_data, runner_data): + (f"_spec-{spec_decoding}" if spec_decoding != "none" else "") ), Fields.SCENARIO_TYPE.value: "agentic-coding", - Fields.RUN_EVAL.value: False, + Fields.RUN_EVAL.value: True, } if kv_offload_backend is not None: entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend @@ -1083,7 +1083,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): ), Fields.DISAGG.value: disagg, Fields.SCENARIO_TYPE.value: "agentic-coding", - Fields.RUN_EVAL.value: False, + Fields.RUN_EVAL.value: True, } if kv_offload_backend is not None: entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend @@ -1116,7 +1116,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): + (f"_spec-{spec_decoding}" if spec_decoding != "none" else "") ), Fields.SCENARIO_TYPE.value: "agentic-coding", - Fields.RUN_EVAL.value: False, + Fields.RUN_EVAL.value: True, } if kv_offload_backend is not None: entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend @@ -1369,7 +1369,7 @@ def main(): # Apply the existing eval policy first, then expand it when requested. if not args.no_evals: - matrix_values = mark_eval_entries(matrix_values) + matrix_values = mark_eval_entries(matrix_values, include_agentic=args.evals_only or args.all_evals) if args.all_evals: matrix_values = mark_all_eval_entries(matrix_values) diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 502a9c1429..1ad4e3c922 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -233,7 +233,7 @@ def test_marks_agentic_entry_for_swebench(self): assert len(marked) == 1 assert marked[0]["conc"] == 64 - def test_default_mode_marks_agentic(self): + def test_default_mode_does_not_mark_agentic(self): matrix_values = [ { "scenario-type": "agentic-coding", @@ -250,10 +250,9 @@ def test_default_mode_marks_agentic(self): result = mark_eval_entries(matrix_values) marked = [e for e in result if e.get("run-eval")] - assert len(marked) == 1, ( - f"Expected 1 agentic entry marked run-eval in default mode, got {len(marked)}" + assert len(marked) == 0, ( + f"Expected 0 agentic entries marked run-eval in default mode, got {len(marked)}" ) - assert marked[0]["conc"] == 64 def test_single_node_skips_eval_entries_below_min_conc(self): """Single-node eval selection should ignore conc values below MIN_EVAL_CONC."""