Add H100 MiniMax-M3 NVMe and tiered AgentX sweep

benchmarks/benchmark_lib.sh

@@ -49,19 +49,19 @@ require_agentic_kv_offload_backend() {
             fi
             return 1
             ;;
-        dram)
+        dram|nvme|dram+nvme)
             if [[ "${KV_OFFLOAD_BACKEND:-}" != "$expected_backend" ]]; then
-                echo "Error: expected KV_OFFLOAD_BACKEND=$expected_backend when KV_OFFLOADING=dram, got '${KV_OFFLOAD_BACKEND:-}'" >&2
+                echo "Error: expected KV_OFFLOAD_BACKEND=$expected_backend when KV_OFFLOADING=$KV_OFFLOADING, got '${KV_OFFLOAD_BACKEND:-}'" >&2
                 exit 1
             fi
-            if [[ ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
-                echo "Error: DRAM KV offloading requires a positive TOTAL_CPU_DRAM_GB capacity" >&2
+            if [[ "$KV_OFFLOADING" != "nvme" && ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
+                echo "Error: $KV_OFFLOADING KV offloading requires a positive TOTAL_CPU_DRAM_GB capacity" >&2
                 exit 1
             fi
             return 0
             ;;
         *)
-            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram)" >&2
+            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram, nvme, dram+nvme)" >&2
             exit 1
             ;;
     esac
@@ -86,18 +86,18 @@ if [[ "$_benchmark_caller" == */agentic/* ||
                 exit 1
             fi
             ;;
-        dram)
+        dram|nvme|dram+nvme)
             if [[ -z "${KV_OFFLOAD_BACKEND:-}" || "${KV_OFFLOAD_BACKEND:-}" == "none" ]]; then
-                echo "Error: KV_OFFLOAD_BACKEND is required when KV_OFFLOADING=dram" >&2
+                echo "Error: KV_OFFLOAD_BACKEND is required when KV_OFFLOADING=$KV_OFFLOADING" >&2
                 exit 1
             fi
-            if [[ ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
-                echo "Error: DRAM KV offloading requires a positive configured TOTAL_CPU_DRAM_GB capacity" >&2
+            if [[ "$KV_OFFLOADING" != "nvme" && ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
+                echo "Error: $KV_OFFLOADING KV offloading requires a positive configured TOTAL_CPU_DRAM_GB capacity" >&2
                 exit 1
             fi
             ;;
         *)
-            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram)" >&2
+            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram, nvme, dram+nvme)" >&2
             exit 1
             ;;
     esac

benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh

@@ -2,7 +2,7 @@
 set -eo pipefail
 set -x
 
-# H100 MiniMax-M3 MXFP8 AgentX with EAGLE3 and optional Mooncake DRAM KV offload.
+# H100 MiniMax-M3 MXFP8 AgentX with EAGLE3 and optional DRAM or NVMe KV offload.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -97,6 +97,29 @@ EOF
         --kv-transfer-config
         '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
     )
+elif [ "$KV_OFFLOADING" = "nvme" ]; then
+    require_agentic_kv_offload_backend vllm-simple
+    : "${NVME_OFFLOAD_DIR:?NVME_OFFLOAD_DIR must be mounted by the H100 launcher}"
+    NVME_OFFLOAD_TOTAL_BYTES=8000000000000
+    NVME_OFFLOAD_PER_RANK_BYTES=$((NVME_OFFLOAD_TOTAL_BYTES / TP))
+    OFFLOAD_ARGS=(
+        --kv-transfer-config
+        "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"kv_offload_backend\":\"disk\",\"disk_path\":\"$NVME_OFFLOAD_DIR/cache.bin\",\"disk_capacity_bytes\":$NVME_OFFLOAD_PER_RANK_BYTES,\"disk_buffer_slots\":4,\"lazy_offload\":false}}"
+    )
+elif [ "$KV_OFFLOADING" = "dram+nvme" ]; then
+    require_agentic_kv_offload_backend vllm-native
+    : "${NVME_OFFLOAD_DIR:?NVME_OFFLOAD_DIR must be mounted by the H100 launcher}"
+    TOTAL_CPU_DRAM_GIB=$((TOTAL_CPU_DRAM_GB * 1000000000 / 1073741824))
+    PER_RANK_GIB=$(((TOTAL_CPU_DRAM_GIB - MODEL_CHECKPOINT_PAGE_CACHE_GIB) / TP - MODEL_CPU_OFFLOAD_GB - MOONCAKE_LOCAL_BUFFER_GIB))
+    if (( PER_RANK_GIB <= 0 )); then
+        echo "Error: CPU DRAM budget is too small for checkpoint cache, model, and DRAM+NVMe KV offload" >&2
+        exit 1
+    fi
+    CPU_OFFLOAD_TOTAL_BYTES=$((PER_RANK_GIB * TP * 1073741824))
+    OFFLOAD_ARGS=(
+        --kv-transfer-config
+        "{\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":$CPU_OFFLOAD_TOTAL_BYTES,\"eviction_policy\":\"lru\",\"secondary_tiers\":[{\"type\":\"fs\",\"root_dir\":\"$NVME_OFFLOAD_DIR\",\"n_read_threads\":32,\"n_write_threads\":16,\"locality\":\"LOCAL\"}]}}"
+    )
 else
     echo "Error: unsupported KV_OFFLOADING='$KV_OFFLOADING'" >&2
     exit 1

configs/nvidia-master.yaml

@@ -7341,7 +7341,7 @@ qwen3.5-fp4-b200-trt-mtp:
       - { tp: 8, ep: 8, dp-attn: true, spec-decoding: "mtp", conc-list: [128, 256, 1024] }
 
 minimaxm3-fp8-h100-vllm-agentic-mtp:
-  image: vllm/vllm-openai:v0.27.1
+  image: ttl.sh/cquil11-vllm-tier-dfb25dbef4-20260828:24h
   model: MiniMaxAI/MiniMax-M3-MXFP8
   model-prefix: minimaxm3
   runner: cluster:h100-dgxc
@@ -7353,8 +7353,10 @@ minimaxm3-fp8-h100-vllm-agentic-mtp:
     # The fast sweep places the resident HBM cliff between c5 and c6.
     - dram-utilization: 0.80
       search-space:
-      - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] }
-      - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [6, 8] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: nvme, kv-offload-backend: { name: vllm-simple }, conc-list: [7, 8, 9, 10, 11, 12, 13, 14] }
+      - { tp: 8, spec-decoding: mtp, kv-offloading: [dram, nvme], kv-offload-backend: { name: vllm-native }, conc-list: [7, 8, 9, 10, 11, 12, 13, 14] }
 
 minimaxm3-fp8-h200-vllm-agentic-mtp:
   image: vllm/vllm-openai:v0.27.1

perf-changelog.yaml

@@ -6576,3 +6576,15 @@
     - "Recipes sourced from srt-slurm (recipes/trtllm/qwen3.5-fp4/inferencex/gb300/{mtp,stp})."
     - "Runner: launch_gb300-nv.sh bumped from NVIDIA/srt-slurm@v1.0.29 to v1.0.72 for the dynamo-trt+qwen3.5+fp4 path."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2730
+
+- config-keys:
+    - minimaxm3-fp8-h100-vllm-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Run the official H100 MiniMax-M3 AgentX sweep on the patched vLLM image that exports cached prompt tokens by physical source tier."
+    - "Retain the no-offload TP8 points at concurrency 1-8 and the Mooncake DRAM points at concurrency 5-10; add NVMe-only points at concurrency 7-14 using SimpleCPUOffloadConnector's disk backend with 8 TB aggregate capacity."
+    - "Add the same concurrency 7-14 range with declarative kv-offloading [dram, nvme], mapped by the vLLM recipe to OffloadingConnector's TieringOffloadingSpec with an LRU DRAM primary tier and node-local filesystem secondary tier."
+    - "Mount a job-scoped directory from the H100 node's native NVMe filesystem into the Pyxis container and remove it during allocation cleanup."
+    - "Collect the vLLM Prometheus endpoint through AIPerf so artifacts include vllm:prompt_tokens_cached_by_source alongside the native KV-offload tiering counters."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2775

runners/launch_h100-dgxc-slurm.sh

@@ -295,7 +295,26 @@ else
         echo "ERROR: failed to resolve H100 Slurm allocation" >&2
         exit 1
     fi
-    trap 'rc=$?; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT
+    NVME_HOST_DIR=""
+    cleanup_allocation() {
+        local rc=$?
+        trap - EXIT INT TERM
+        if [[ -n "$NVME_HOST_DIR" ]]; then
+            srun --jobid="$JOB_ID" bash -c "rm -rf -- '$NVME_HOST_DIR'" 2>/dev/null || true
+        fi
+        scancel "$JOB_ID" 2>/dev/null || true
+        exit "$rc"
+    }
+    trap cleanup_allocation EXIT INT TERM
+
+    NVME_CONTAINER_MOUNT=""
+    if [[ "${KV_OFFLOADING:-none}" == "nvme" || "${KV_OFFLOADING:-none}" == "dram+nvme" ]]; then
+        NVME_HOST_ROOT="/mnt/numa0/enroot/cache/group-$(id -g)"
+        NVME_HOST_DIR="$NVME_HOST_ROOT/inferencex-kv-$JOB_ID"
+        srun --jobid="$JOB_ID" bash -c "set -e; test -w '$NVME_HOST_ROOT'; mkdir -m 700 '$NVME_HOST_DIR'; findmnt -T '$NVME_HOST_DIR'"
+        NVME_CONTAINER_MOUNT=",$NVME_HOST_DIR:/kv-offload"
+        export NVME_OFFLOAD_DIR=/kv-offload
+    fi
 
     # flock-serialize the enroot import so concurrent sweep jobs on the same
     # shared NFS path don't race each other into 'File already exists' (race
@@ -317,7 +336,7 @@ else
 
     srun --jobid=$JOB_ID \
         --container-image=$SQUASH_FILE \
-        --container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache \
+        --container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache$NVME_CONTAINER_MOUNT \
         --no-container-mount-home \
         --container-workdir=/workspace/ \
         --no-container-entrypoint --export=ALL,PORT=8888,AIPERF_DATASET_MMAP_CACHE_DIR=/aiperf_mmap_cache \

utils/matrix_logic/generate_sweep_configs.py

@@ -322,7 +322,7 @@ def agentic_dram_offload_gb(
       budgeted separately if it ever gains its own pool).
     """
     kv_offloading = benchmark.get(Fields.KV_OFFLOADING.value, "none")
-    if kv_offloading != "dram":
+    if kv_offloading != "dram" and kv_offloading != ["dram", "nvme"]:
         return 0
 
     available_mib = min(
@@ -356,13 +356,21 @@ def agentic_dram_offload_gb(
 
 
 def agentic_kv_offload_suffix(
-    kv_offloading: str,
+    kv_offloading: str | list[str],
     kv_offload_backend: dict | None,
 ) -> str:
     """Return a compact exp-name suffix for agentic KV offload settings."""
     if kv_offloading == "none":
         return "kvnone"
-    return f"kv{kv_offloading}-{kv_offload_backend['name']}"
+    mode = "+".join(kv_offloading) if isinstance(kv_offloading, list) else kv_offloading
+    return f"kv{mode}-{kv_offload_backend['name']}"
+
+
+def agentic_kv_offload_runtime_value(kv_offloading: str | list[str]) -> str:
+    """Convert declarative tier lists into the workflow's string input."""
+    if isinstance(kv_offloading, list):
+        return "+".join(kv_offloading)
+    return kv_offloading
 
 
 def multinode_agentic_exp_name(
@@ -1029,7 +1037,7 @@ def generate_full_sweep(args, all_config_data, runner_data):
                                 Fields.PREFILL.value: prefill,
                                 Fields.DECODE.value: decode,
                                 Fields.CONC.value: conc_batch,
-                                Fields.KV_OFFLOADING.value: kv_offloading,
+                                Fields.KV_OFFLOADING.value: agentic_kv_offload_runtime_value(kv_offloading),
                                 Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb,
                                 Fields.DURATION.value: duration,
                                 Fields.EXP_NAME.value: multinode_agentic_exp_name(
@@ -1066,7 +1074,7 @@ def generate_full_sweep(args, all_config_data, runner_data):
                                 Fields.DP_ATTN.value: dp_attn if dp_attn is not None else False,
                                 Fields.SPEC_DECODING.value: spec_decoding,
                                 Fields.CONC.value: conc,
-                                Fields.KV_OFFLOADING.value: kv_offloading,
+                                Fields.KV_OFFLOADING.value: agentic_kv_offload_runtime_value(kv_offloading),
                                 Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb,
                                 Fields.DURATION.value: duration,
                                 Fields.EXP_NAME.value: (
@@ -1328,7 +1336,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None):
                                 Fields.PREFILL.value: prefill,
                                 Fields.DECODE.value: decode,
                                 Fields.CONC.value: conc_batch,
-                                Fields.KV_OFFLOADING.value: kv_offloading,
+                                Fields.KV_OFFLOADING.value: agentic_kv_offload_runtime_value(kv_offloading),
                                 Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb,
                                 Fields.DURATION.value: duration,
                                 Fields.EXP_NAME.value: multinode_agentic_exp_name(
@@ -1364,7 +1372,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None):
                                 Fields.DP_ATTN.value: dp_attn if dp_attn is not None else False,
                                 Fields.SPEC_DECODING.value: spec_decoding,
                                 Fields.CONC.value: conc,
-                                Fields.KV_OFFLOADING.value: kv_offloading,
+                                Fields.KV_OFFLOADING.value: agentic_kv_offload_runtime_value(kv_offloading),
                                 Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb,
                                 Fields.DURATION.value: duration,
                                 Fields.EXP_NAME.value: (

utils/matrix_logic/test_generate_sweep_configs.py

@@ -2385,6 +2385,56 @@ def test_agentic_node_dram_uses_explicit_gpu_count(self, sample_runner_config):
         }
         assert all(entry["duration"] == 3600 for entry in result)
 
+    def test_multi_tier_agentic_uses_dram_budget_and_distinct_name(
+        self, sample_runner_config
+    ):
+        config = {
+            "dsv4-b300-agentic": {
+                "image": "vllm/vllm-openai:v0.23.0",
+                "model": "deepseek-ai/DeepSeek-V4-Pro",
+                "model-prefix": "dsv4",
+                "precision": "fp4",
+                "framework": "vllm",
+                "runner": "cluster:b300-nv",
+                "multinode": False,
+                "scenarios": {
+                    "agentic-coding": [{
+                        "dram-utilization": 0.80,
+                        "search-space": [
+                            {
+                                "tp": 8,
+                                "kv-offloading": "nvme",
+                                "kv-offload-backend": {"name": "vllm-simple"},
+                                "conc-list": [7],
+                            },
+                            {
+                                "tp": 8,
+                                "kv-offloading": ["dram", "nvme"],
+                                "kv-offload-backend": {"name": "vllm-native"},
+                                "conc-list": [7],
+                            },
+                        ],
+                    }],
+                },
+            },
+        }
+        args = argparse.Namespace(
+            config_keys=["dsv4-b300-agentic"],
+            seq_lens=None,
+            conc=None,
+            scenario_type=["agentic-coding"],
+            runner_node_filter=None,
+        )
+
+        result = generate_test_config_sweep(args, config, sample_runner_config)
+
+        assert [entry["kv-offloading"] for entry in result] == ["nvme", "dram+nvme"]
+        assert [entry["total-cpu-dram-gb"] for entry in result] == [0, 2399]
+        assert [entry["exp-name"] for entry in result] == [
+            "dsv4_tp8_conc7_kvnvme-vllm-simple",
+            "dsv4_tp8_conc7_kvdram+nvme-vllm-native",
+        ]
+
     def test_agentic_node_dram_rejects_tp_above_runner_gpus(self, sample_runner_config):
         config = {
             "dsv4-b300-agentic": {

utils/matrix_logic/test_validation.py

@@ -530,6 +530,31 @@ def test_dram_kv_offload_requires_dram_utilization(self):
                 }],
             })
 
+    def test_multi_tier_kv_offload_requires_dram_utilization(self):
+        with pytest.raises(Exception, match="dram-utilization"):
+            AgenticCodingConfig(**{
+                "search-space": [{
+                    "tp": 8,
+                    "kv-offloading": ["dram", "nvme"],
+                    "kv-offload-backend": {"name": "vllm-native"},
+                    "conc-list": [7, 8],
+                }],
+            })
+
+    def test_multi_tier_kv_offload_accepts_dram_capacity_config(self):
+        config = AgenticCodingConfig(**{
+            "dram-utilization": 0.99,
+            "search-space": [{
+                "tp": 8,
+                "kv-offloading": ["dram", "nvme"],
+                "kv-offload-backend": {"name": "vllm-native"},
+                "conc-list": [7, 8],
+            }],
+        })
+
+        assert config.search_space[0].kv_offloading == ["dram", "nvme"]
+        assert config.dram_utilization == 0.99
+
     def test_agentic_search_space_rejects_total_cpu_dram_gb(self):
         with pytest.raises(Exception, match="total-cpu-dram-gb"):
             AgenticCodingSearchSpaceEntry(**{

utils/matrix_logic/validation.py

@@ -14,6 +14,11 @@
 
 CLUSTER_LABEL_PREFIX = "cluster:"
 DEFAULT_AGENTIC_DURATION_SECONDS = 3600
+KVOffloadingTier = Literal["dram", "nvme"]
+KVOffloadingConfig = Union[
+    Literal["none", "dram", "nvme"],
+    List[KVOffloadingTier],
+]
 
 """
     The below class defines the field names expected to be present in the JSON entries
@@ -318,7 +323,7 @@ class SingleNodeAgenticMatrixEntry(BaseModel):
         default="none", alias=Fields.SPEC_DECODING.value
     )
     conc: int
-    kv_offloading: Literal["none", "dram"] = Field(
+    kv_offloading: Literal["none", "dram", "nvme", "dram+nvme"] = Field(
         alias=Fields.KV_OFFLOADING.value
     )
     kv_offload_backend: Optional[KVOffloadBackendMetadata] = Field(
@@ -513,7 +518,13 @@ def _validate_kv_offload_fields(self):
                 f"{Fields.KV_OFFLOADING.value}"
             )
         return self
-    if self.kv_offloading == "none":
+    if isinstance(self.kv_offloading, list):
+        if self.kv_offloading != ["dram", "nvme"]:
+            raise ValueError(
+                f"The only supported multi-tier {Fields.KV_OFFLOADING.value} "
+                "configuration is ['dram', 'nvme']"
+            )
+    elif self.kv_offloading == "none":
         if backend is not None:
             raise ValueError(
                 f"{Fields.KV_OFFLOAD_BACKEND.value} can only be set when "
@@ -644,7 +655,7 @@ class AgenticCodingSearchSpaceEntry(BaseModel):
     decode: Optional[WorkerConfig] = None
     num_nodes: Optional[int] = Field(
         default=None, alias=Fields.NUM_NODES.value, gt=0, strict=True)
-    kv_offloading: Optional[Literal["none", "dram"]] = Field(
+    kv_offloading: Optional[KVOffloadingConfig] = Field(
         default=None, alias=Fields.KV_OFFLOADING.value
     )
     kv_offload_backend: Optional[KVOffloadBackendMetadata] = Field(
@@ -726,7 +737,7 @@ class AgenticCodingConfig(BaseModel):
     @model_validator(mode='after')
     def validate_dram_offload_capacity(self):
         for entry in self.search_space:
-            if entry.kv_offloading != "dram":
+            if entry.kv_offloading != "dram" and entry.kv_offloading != ["dram", "nvme"]:
                 continue
             if self.dram_utilization is None:
                 raise ValueError(