Add H100 MiniMax-M3 NVMe and tiered AgentX sweep

benchmarks/benchmark_lib.sh

@@ -49,19 +49,19 @@ require_agentic_kv_offload_backend() {
             fi
             return 1
             ;;
-        dram)
+        dram|nvme|dram+nvme)
             if [[ "${KV_OFFLOAD_BACKEND:-}" != "$expected_backend" ]]; then
-                echo "Error: expected KV_OFFLOAD_BACKEND=$expected_backend when KV_OFFLOADING=dram, got '${KV_OFFLOAD_BACKEND:-}'" >&2
+                echo "Error: expected KV_OFFLOAD_BACKEND=$expected_backend when KV_OFFLOADING=$KV_OFFLOADING, got '${KV_OFFLOAD_BACKEND:-}'" >&2
                 exit 1
             fi
-            if [[ ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
-                echo "Error: DRAM KV offloading requires a positive TOTAL_CPU_DRAM_GB capacity" >&2
+            if [[ "$KV_OFFLOADING" != "nvme" && ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
+                echo "Error: $KV_OFFLOADING KV offloading requires a positive TOTAL_CPU_DRAM_GB capacity" >&2
                 exit 1
             fi
             return 0
             ;;
         *)
-            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram)" >&2
+            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram, nvme, dram+nvme)" >&2
             exit 1
             ;;
     esac
@@ -86,18 +86,18 @@ if [[ "$_benchmark_caller" == */agentic/* ||
                 exit 1
             fi
             ;;
-        dram)
+        dram|nvme|dram+nvme)
             if [[ -z "${KV_OFFLOAD_BACKEND:-}" || "${KV_OFFLOAD_BACKEND:-}" == "none" ]]; then
-                echo "Error: KV_OFFLOAD_BACKEND is required when KV_OFFLOADING=dram" >&2
+                echo "Error: KV_OFFLOAD_BACKEND is required when KV_OFFLOADING=$KV_OFFLOADING" >&2
                 exit 1
             fi
-            if [[ ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
-                echo "Error: DRAM KV offloading requires a positive configured TOTAL_CPU_DRAM_GB capacity" >&2
+            if [[ "$KV_OFFLOADING" != "nvme" && ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
+                echo "Error: $KV_OFFLOADING KV offloading requires a positive configured TOTAL_CPU_DRAM_GB capacity" >&2
                 exit 1
             fi
             ;;
         *)
-            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram)" >&2
+            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram, nvme, dram+nvme)" >&2
             exit 1
             ;;
     esac

benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh

@@ -2,7 +2,7 @@
 set -eo pipefail
 set -x
 
-# H100 MiniMax-M3 MXFP8 AgentX with EAGLE3 and optional Mooncake DRAM KV offload.
+# H100 MiniMax-M3 MXFP8 AgentX with EAGLE3 and optional DRAM or NVMe KV offload.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -47,6 +47,9 @@ cleanup_services() {
     set +e
     stop_background_process_tree "$SERVER_PID" "vLLM server" 60
     stop_background_process_tree "$MOONCAKE_MASTER_PID" "Mooncake master" 30
+    if [[ -n "${NVME_OFFLOAD_DIR:-}" ]]; then
+        find "$NVME_OFFLOAD_DIR" -mindepth 1 -delete
+    fi
     exit "$exit_code"
 }
 trap cleanup_services EXIT
@@ -97,6 +100,30 @@ EOF
         --kv-transfer-config
         '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
     )
+elif [ "$KV_OFFLOADING" = "nvme" ]; then
+    require_agentic_kv_offload_backend vllm-simple
+    : "${NVME_OFFLOAD_DIR:?NVME_OFFLOAD_DIR must be mounted by the H100 launcher}"
+    NVME_OFFLOAD_TOTAL_BYTES=8000000000000
+    NVME_OFFLOAD_PER_RANK_BYTES=$((NVME_OFFLOAD_TOTAL_BYTES / TP))
+    # vLLM appends .rank_<CUDA device index> to give each TP rank its own file.
+    OFFLOAD_ARGS=(
+        --kv-transfer-config
+        "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"kv_offload_backend\":\"disk\",\"disk_path\":\"$NVME_OFFLOAD_DIR/cache.bin\",\"disk_capacity_bytes\":$NVME_OFFLOAD_PER_RANK_BYTES,\"disk_buffer_slots\":4,\"lazy_offload\":false}}"
+    )
+elif [ "$KV_OFFLOADING" = "dram+nvme" ]; then
+    require_agentic_kv_offload_backend vllm-native
+    : "${NVME_OFFLOAD_DIR:?NVME_OFFLOAD_DIR must be mounted by the H100 launcher}"
+    TOTAL_CPU_DRAM_GIB=$((TOTAL_CPU_DRAM_GB * 1000000000 / 1073741824))
+    PER_RANK_GIB=$(((TOTAL_CPU_DRAM_GIB - MODEL_CHECKPOINT_PAGE_CACHE_GIB) / TP - MODEL_CPU_OFFLOAD_GB - MOONCAKE_LOCAL_BUFFER_GIB))
+    if (( PER_RANK_GIB <= 0 )); then
+        echo "Error: CPU DRAM budget is too small for checkpoint cache, model, and DRAM+NVMe KV offload" >&2
+        exit 1
+    fi
+    CPU_OFFLOAD_TOTAL_BYTES=$((PER_RANK_GIB * TP * 1073741824))
+    OFFLOAD_ARGS=(
+        --kv-transfer-config
+        "{\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":$CPU_OFFLOAD_TOTAL_BYTES,\"eviction_policy\":\"lru\",\"secondary_tiers\":[{\"type\":\"fs\",\"root_dir\":\"$NVME_OFFLOAD_DIR\",\"n_read_threads\":32,\"n_write_threads\":16,\"locality\":\"LOCAL\"}]}}"
+    )
 else
     echo "Error: unsupported KV_OFFLOADING='$KV_OFFLOADING'" >&2
     exit 1

configs/nvidia-master.yaml

@@ -7341,7 +7341,7 @@ qwen3.5-fp4-b200-trt-mtp:
       - { tp: 8, ep: 8, dp-attn: true, spec-decoding: "mtp", conc-list: [128, 256, 1024] }
 
 minimaxm3-fp8-h100-vllm-agentic-mtp:
-  image: vllm/vllm-openai:v0.27.1
+  image: ttl.sh/cquil11-vllm-tier-dfb25dbef4-20260828:24h
   model: MiniMaxAI/MiniMax-M3-MXFP8
   model-prefix: minimaxm3
   runner: cluster:h100-dgxc
@@ -7353,8 +7353,10 @@ minimaxm3-fp8-h100-vllm-agentic-mtp:
     # The fast sweep places the resident HBM cliff between c5 and c6.
     - dram-utilization: 0.80
       search-space:
-      - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] }
-      - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [6, 8] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: nvme, kv-offload-backend: { name: vllm-simple }, conc-list: [7, 8, 9, 10, 11, 12, 13, 14] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: [dram, nvme], kv-offload-backend: { name: vllm-native }, conc-list: [7, 8, 9, 10, 11, 12, 13, 14] }
 
 minimaxm3-fp8-h200-vllm-agentic-mtp:
   image: vllm/vllm-openai:v0.27.1

docs/recovery-results-procedures.md

@@ -266,6 +266,19 @@ After pushing, never rebase, locally squash, amend, or force-push this carrier c
 
 Canonical source: [complete failed-ingest recovery command](https://github.com/SemiAnalysisAI/InferenceX/blob/0c28706b33d4a796b82f6f9c3594c19c46365575/.claude/commands/recover-failed-ingest.md).
 
+## H100 offload cleanup stalls
+
+An H100 AgentX benchmark can finish and write all result files while the GitHub job remains in `Launch job script`. Check the launcher process tree on the login node. A long-running child matching either of these commands means teardown, rather than the benchmark, is stalled:
+
+```text
+srun --jobid=<id> bash -c find /dev/shm ... vllm_offload_*.mmap ... -delete
+srun --jobid=<id> bash -c rm -rf -- /mnt/numa0/.../inferencex-kv-<id>
+```
+
+Preserve and verify the result files before intervening. The H100 launcher bounds each pre-run and post-run offload cleanup step with `H100_OFFLOAD_CLEANUP_TIMEOUT_S` (120 seconds by default), followed by a 15-second forced-termination window. A post-run timeout is best-effort: the launcher cancels the allocation and preserves the benchmark exit status so artifact uploads can proceed. A pre-run timeout fails closed because stale shared memory or NVMe state can invalidate the next benchmark.
+
+For an already-running job created before this guard existed, terminate only the stuck login-node `srun` client after confirming the benchmark outputs are complete. The launcher's cleanup trap then continues to `scancel` the allocation. Do not cancel the GitHub job or delete its workspace; doing so can discard uploadable evidence.
+
 ## AMD root-owned workspace prevention and recovery
 
 ### Prevent recurrence

docs/recovery-results-procedures_zh.md

@@ -266,6 +266,19 @@ git diff --check origin/main...HEAD
 
 权威来源：[完整失败摄取恢复命令](https://github.com/SemiAnalysisAI/InferenceX/blob/0c28706b33d4a796b82f6f9c3594c19c46365575/.claude/commands/recover-failed-ingest.md)。
 
+## H100 offload 清理停滞
+
+H100 AgentX 基准可能已经完成并写出所有结果文件，但 GitHub 任务仍停留在 `Launch job script`。请在登录节点检查 launcher 进程树。若以下任一命令的子进程长时间运行，说明停滞发生在 teardown，而不是基准本身：
+
+```text
+srun --jobid=<id> bash -c find /dev/shm ... vllm_offload_*.mmap ... -delete
+srun --jobid=<id> bash -c rm -rf -- /mnt/numa0/.../inferencex-kv-<id>
+```
+
+干预前必须保留并验证结果文件。H100 launcher 使用 `H100_OFFLOAD_CLEANUP_TIMEOUT_S` 限制每个运行前和运行后的 offload 清理步骤；默认超时为 120 秒，随后保留 15 秒强制终止窗口。运行后超时按 best-effort 处理：launcher 会取消 allocation，并保留基准退出状态，使 artifact 上传可以继续。运行前超时则会 fail closed，因为残留的共享内存或 NVMe 状态可能使下一次基准结果无效。
+
+对于在该保护逻辑加入前已经启动的任务，只有在确认基准输出完整后，才能终止登录节点上卡住的 `srun` client。随后 launcher 的 cleanup trap 会继续执行 `scancel` 来释放 allocation。不要取消 GitHub 任务，也不要删除其工作区，否则可能丢失仍可上传的证据。
+
 ## AMD root-owned 工作区的预防与恢复
 
 ### 防止复发

perf-changelog.yaml

@@ -6576,3 +6576,15 @@
     - "Recipes sourced from srt-slurm (recipes/trtllm/qwen3.5-fp4/inferencex/gb300/{mtp,stp})."
     - "Runner: launch_gb300-nv.sh bumped from NVIDIA/srt-slurm@v1.0.29 to v1.0.72 for the dynamo-trt+qwen3.5+fp4 path."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2730
+
+- config-keys:
+    - minimaxm3-fp8-h100-vllm-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Run the official H100 MiniMax-M3 AgentX sweep on the patched vLLM image that exports cached prompt tokens by physical source tier."
+    - "Retain the no-offload TP8 points at concurrency 1-8 and the Mooncake DRAM points at concurrency 5-10; add NVMe-only points at concurrency 7-14 using SimpleCPUOffloadConnector's disk backend with 8 TB aggregate capacity."
+    - "Add the same concurrency 7-14 range with declarative kv-offloading [dram, nvme], mapped by the vLLM recipe to OffloadingConnector's TieringOffloadingSpec with an LRU DRAM primary tier and node-local filesystem secondary tier."
+    - "Mount a job-scoped directory from the H100 node's native NVMe filesystem into the Pyxis container and remove it during allocation cleanup."
+    - "Collect the vLLM Prometheus endpoint through AIPerf so artifacts include vllm:prompt_tokens_cached_by_source alongside the native KV-offload tiering counters."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2775

runners/launch_h100-dgxc-slurm.sh

@@ -289,13 +289,43 @@ else
 
     export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}"
 
-    salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT --gres=gpu:$GPU_COUNT --exclusive --time=180 --no-shell --job-name="$RUNNER_NAME"
+    SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-300}"
+    SALLOC_ARGS=(
+        --partition="$SLURM_PARTITION"
+        --account="$SLURM_ACCOUNT"
+        --gres="gpu:$GPU_COUNT"
+        --exclusive
+        --time="$SALLOC_TIME_LIMIT"
+        --no-shell
+        --job-name="$RUNNER_NAME"
+    )
+    salloc "${SALLOC_ARGS[@]}"
     JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1)
     if [[ -z "$JOB_ID" ]]; then
         echo "ERROR: failed to resolve H100 Slurm allocation" >&2
         exit 1
     fi
-    trap 'rc=$?; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT
+    cleanup_allocation() {
+        local rc=$?
+        trap - EXIT INT TERM
+        scancel "$JOB_ID" 2>/dev/null || true
+        exit "$rc"
+    }
+    trap cleanup_allocation EXIT INT TERM
+
+    NVME_CONTAINER_MOUNT=""
+    if [[ "${KV_OFFLOADING:-none}" == "nvme" || "${KV_OFFLOADING:-none}" == "dram+nvme" ]]; then
+        NVME_HOST_ROOT="/mnt/numa0/enroot/cache/group-$(id -g)"
+        NVME_HOST_DIR="$NVME_HOST_ROOT/inferencex-kv-$JOB_ID"
+        srun --jobid="$JOB_ID" bash -c "
+            set -e
+            test -w '$NVME_HOST_ROOT'
+            mkdir -m 700 '$NVME_HOST_DIR'
+            findmnt -T '$NVME_HOST_DIR'
+        "
+        NVME_CONTAINER_MOUNT=",$NVME_HOST_DIR:/kv-offload"
+        export NVME_OFFLOAD_DIR=/kv-offload
+    fi
 
     # flock-serialize the enroot import so concurrent sweep jobs on the same
     # shared NFS path don't race each other into 'File already exists' (race
@@ -317,12 +347,10 @@ else
 
     srun --jobid=$JOB_ID \
         --container-image=$SQUASH_FILE \
-        --container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache \
+        --container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache$NVME_CONTAINER_MOUNT \
         --no-container-mount-home \
         --container-workdir=/workspace/ \
         --no-container-entrypoint --export=ALL,PORT=8888,AIPERF_DATASET_MMAP_CACHE_DIR=/aiperf_mmap_cache \
         bash benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_h100${SPEC_SUFFIX}.sh
 
-    scancel $JOB_ID
-
 fi

utils/matrix_logic/generate_sweep_configs.py

@@ -322,7 +322,7 @@ def agentic_dram_offload_gb(
       budgeted separately if it ever gains its own pool).
     """
     kv_offloading = benchmark.get(Fields.KV_OFFLOADING.value, "none")
-    if kv_offloading != "dram":
+    if kv_offloading != "dram" and kv_offloading != ["dram", "nvme"]:
         return 0
 
     available_mib = min(
@@ -356,13 +356,21 @@ def agentic_dram_offload_gb(
 
 
 def agentic_kv_offload_suffix(
-    kv_offloading: str,
+    kv_offloading: str | list[str],
     kv_offload_backend: dict | None,
 ) -> str:
     """Return a compact exp-name suffix for agentic KV offload settings."""
     if kv_offloading == "none":
         return "kvnone"
-    return f"kv{kv_offloading}-{kv_offload_backend['name']}"
+    mode = "+".join(kv_offloading) if isinstance(kv_offloading, list) else kv_offloading
+    return f"kv{mode}-{kv_offload_backend['name']}"
+
+
+def agentic_kv_offload_runtime_value(kv_offloading: str | list[str]) -> str:
+    """Convert declarative tier lists into the workflow's string input."""
+    if isinstance(kv_offloading, list):
+        return "+".join(kv_offloading)
+    return kv_offloading
 
 
 def multinode_agentic_exp_name(
@@ -1029,7 +1037,7 @@ def generate_full_sweep(args, all_config_data, runner_data):
                                 Fields.PREFILL.value: prefill,
                                 Fields.DECODE.value: decode,
                                 Fields.CONC.value: conc_batch,
-                                Fields.KV_OFFLOADING.value: kv_offloading,
+                                Fields.KV_OFFLOADING.value: agentic_kv_offload_runtime_value(kv_offloading),
                                 Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb,
                                 Fields.DURATION.value: duration,
                                 Fields.EXP_NAME.value: multinode_agentic_exp_name(
@@ -1066,7 +1074,7 @@ def generate_full_sweep(args, all_config_data, runner_data):
                                 Fields.DP_ATTN.value: dp_attn if dp_attn is not None else False,
                                 Fields.SPEC_DECODING.value: spec_decoding,
                                 Fields.CONC.value: conc,
-                                Fields.KV_OFFLOADING.value: kv_offloading,
+                                Fields.KV_OFFLOADING.value: agentic_kv_offload_runtime_value(kv_offloading),
                                 Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb,
                                 Fields.DURATION.value: duration,
                                 Fields.EXP_NAME.value: (
@@ -1328,7 +1336,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None):
                                 Fields.PREFILL.value: prefill,
                                 Fields.DECODE.value: decode,
                                 Fields.CONC.value: conc_batch,
-                                Fields.KV_OFFLOADING.value: kv_offloading,
+                                Fields.KV_OFFLOADING.value: agentic_kv_offload_runtime_value(kv_offloading),
                                 Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb,
                                 Fields.DURATION.value: duration,
                                 Fields.EXP_NAME.value: multinode_agentic_exp_name(
@@ -1364,7 +1372,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None):
                                 Fields.DP_ATTN.value: dp_attn if dp_attn is not None else False,
                                 Fields.SPEC_DECODING.value: spec_decoding,
                                 Fields.CONC.value: conc,
-                                Fields.KV_OFFLOADING.value: kv_offloading,
+                                Fields.KV_OFFLOADING.value: agentic_kv_offload_runtime_value(kv_offloading),
                                 Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb,
                                 Fields.DURATION.value: duration,
                                 Fields.EXP_NAME.value: (

utils/matrix_logic/test_generate_sweep_configs.py

@@ -2385,6 +2385,56 @@ def test_agentic_node_dram_uses_explicit_gpu_count(self, sample_runner_config):
         }
         assert all(entry["duration"] == 3600 for entry in result)
 
+    def test_multi_tier_agentic_uses_dram_budget_and_distinct_name(
+        self, sample_runner_config
+    ):
+        config = {
+            "dsv4-b300-agentic": {
+                "image": "vllm/vllm-openai:v0.23.0",
+                "model": "deepseek-ai/DeepSeek-V4-Pro",
+                "model-prefix": "dsv4",
+                "precision": "fp4",
+                "framework": "vllm",
+                "runner": "cluster:b300-nv",
+                "multinode": False,
+                "scenarios": {
+                    "agentic-coding": [{
+                        "dram-utilization": 0.80,
+                        "search-space": [
+                            {
+                                "tp": 8,
+                                "kv-offloading": "nvme",
+                                "kv-offload-backend": {"name": "vllm-simple"},
+                                "conc-list": [7],
+                            },
+                            {
+                                "tp": 8,
+                                "kv-offloading": ["dram", "nvme"],
+                                "kv-offload-backend": {"name": "vllm-native"},
+                                "conc-list": [7],
+                            },
+                        ],
+                    }],
+                },
+            },
+        }
+        args = argparse.Namespace(
+            config_keys=["dsv4-b300-agentic"],
+            seq_lens=None,
+            conc=None,
+            scenario_type=["agentic-coding"],
+            runner_node_filter=None,
+        )
+
+        result = generate_test_config_sweep(args, config, sample_runner_config)
+
+        assert [entry["kv-offloading"] for entry in result] == ["nvme", "dram+nvme"]
+        assert [entry["total-cpu-dram-gb"] for entry in result] == [0, 2399]
+        assert [entry["exp-name"] for entry in result] == [
+            "dsv4_tp8_conc7_kvnvme-vllm-simple",
+            "dsv4_tp8_conc7_kvdram+nvme-vllm-native",
+        ]
+
     def test_agentic_node_dram_rejects_tp_above_runner_gpus(self, sample_runner_config):
         config = {
             "dsv4-b300-agentic": {