cquil11 · Sep 9, 2026 · Sep 9, 2026 · Sep 9, 2026 · Sep 9, 2026 · Sep 9, 2026
@@ -43,6 +43,7 @@ require_agentic_kv_offload_none() {
 
 require_agentic_kv_offload_backend() {
     local expected_backend="$1"
+    local supported_modes="${2:-dram}"
     if [[ -z "${KV_OFFLOADING+x}" || -z "$KV_OFFLOADING" ]]; then
         echo "Error: KV_OFFLOADING must be set for agentic benchmarks" >&2
         exit 1
@@ -55,19 +56,23 @@ require_agentic_kv_offload_backend() {
             fi
             return 1
             ;;
-        dram)
+        dram|nvme|dram+nvme)
+            if [[ " $supported_modes " != *" $KV_OFFLOADING "* ]]; then
+                echo "Error: this recipe does not support $KV_OFFLOADING with $expected_backend" >&2
+                exit 1
+            fi
             if [[ "${KV_OFFLOAD_BACKEND:-}" != "$expected_backend" ]]; then
-                echo "Error: expected KV_OFFLOAD_BACKEND=$expected_backend when KV_OFFLOADING=dram, got '${KV_OFFLOAD_BACKEND:-}'" >&2
+                echo "Error: expected KV_OFFLOAD_BACKEND=$expected_backend when KV_OFFLOADING=$KV_OFFLOADING, got '${KV_OFFLOAD_BACKEND:-}'" >&2
                 exit 1
             fi
-            if [[ ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
+            if [[ "$KV_OFFLOADING" != "nvme" && ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
                 echo "Error: DRAM KV offloading requires a positive TOTAL_CPU_DRAM_GB capacity" >&2
                 exit 1
             fi
             return 0
             ;;
         *)
-            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram)" >&2
+            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected none, dram, nvme, or dram+nvme)" >&2
             exit 1
             ;;
     esac
@@ -108,18 +113,18 @@ if [[ "$_benchmark_caller" == */agentic/* ||
                 exit 1
             fi
             ;;
-        dram)
+        dram|nvme|dram+nvme)
             if [[ -z "${KV_OFFLOAD_BACKEND:-}" || "${KV_OFFLOAD_BACKEND:-}" == "none" ]]; then
-                echo "Error: KV_OFFLOAD_BACKEND is required when KV_OFFLOADING=dram" >&2
+                echo "Error: KV_OFFLOAD_BACKEND is required when KV_OFFLOADING=$KV_OFFLOADING" >&2
                 exit 1
             fi
-            if [[ ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
+            if [[ "$KV_OFFLOADING" != "nvme" && ! "${TOTAL_CPU_DRAM_GB:-}" =~ ^[1-9][0-9]*$ ]]; then
                 echo "Error: DRAM KV offloading requires a positive configured TOTAL_CPU_DRAM_GB capacity" >&2
                 exit 1
             fi
             ;;
         *)
-            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected one of: none, dram)" >&2
+            echo "Error: unsupported KV_OFFLOADING value '$KV_OFFLOADING' (expected none, dram, nvme, or dram+nvme)" >&2
             exit 1
             ;;
     esac

@@ -11,15 +11,15 @@ set -x
 #   --speculative-config: synthetic acceptance length 2.49 (throughput) vs real MTP (EVAL_ONLY); see the SPEC_CONFIG block
 #   cudagraph capture sizes expressed in TOKENS (see the capture block below).
 #
-# The throughput sweep uses DEP8 with SimpleCPUOffloadConnector only. The recipe
+# The recipe supports TP8 and DEP8 with explicit KV offload backends. It
 # uses FP8 KV cache, sparse DeepSeek-V4 FlashInfer attention with an FP4 indexer
 # cache, mega-MoE, long-prefill chunking, and FULL_DECODE_ONLY CUDA graphs with
 # every decode batch captured explicitly.
 #
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
 #
-# DEP8 offloads KV to host DRAM with KV_OFFLOAD_BACKEND=vllm-simple.
+# Native offloading supports DRAM or DRAM+NVMe; Simple supports DRAM or NVMe.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -101,7 +101,7 @@ export VLLM_ENGINE_READY_TIMEOUT_S=3600
 # store mask. 32k matches the trace-replay tuning validated for this workload.
 export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768
 export VLLM_USE_V2_MODEL_RUNNER=1
-export VLLM_USE_RUST_FRONTEND=1
+export VLLM_USE_RUST_FRONTEND=0
 export VLLM_DSV4_MEGA_FP8_COMBINE=1
 export VLLM_RPC_TIMEOUT=600000
 
@@ -121,7 +121,7 @@ case "$KV_OFFLOAD_BACKEND" in
         require_agentic_kv_offload_none
         ;;
     vllm-simple)
-        require_agentic_kv_offload_backend vllm-simple
+        require_agentic_kv_offload_backend vllm-simple "dram nvme"
         CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT ))
         # Identical prefixes must hash to identical block keys across DP ranks.
         export PYTHONHASHSEED=42
@@ -137,11 +137,28 @@ case "$KV_OFFLOAD_BACKEND" in
 }
 EOF
 )
+        if [ "$KV_OFFLOADING" = "nvme" ]; then
+            : "${NVME_OFFLOAD_DIR:?NVMe cache directory must be mounted by the launcher}"
+            OFFLOAD_CONFIG="{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"kv_offload_backend\":\"disk\",\"disk_path\":\"$NVME_OFFLOAD_DIR/cache.bin\",\"disk_capacity_bytes\":$((1000000000000 / GPU_COUNT)),\"disk_buffer_slots\":4,\"lazy_offload\":false}}"
+        fi
         OFFLOAD_ARGS=(
             --kv-transfer-config
             "$OFFLOAD_CONFIG"
         )
         ;;
+    vllm-native)
+        require_agentic_kv_offload_backend vllm-native "dram dram+nvme"
+        export PYTHONHASHSEED=42
+        SECONDARY_TIERS='[]'
+        if [ "$KV_OFFLOADING" = "dram+nvme" ]; then
+            : "${NVME_OFFLOAD_DIR:?NVMe cache directory must be mounted by the launcher}"
+            SECONDARY_TIERS="[{\"type\":\"fs\",\"root_dir\":\"$NVME_OFFLOAD_DIR\",\"locality\":\"LOCAL\"}]"
+        fi
+        OFFLOAD_ARGS=(
+            --kv-transfer-config
+            "{\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":$((TOTAL_CPU_DRAM_GB * 1000000000)),\"secondary_tiers\":$SECONDARY_TIERS}}"
+        )
+        ;;
     mooncake)
         require_agentic_kv_offload_backend mooncake
         # Embedded mode contributes one segment per GPU rank to a shared
@@ -280,7 +297,8 @@ echo "Starting vllm server..."
 export TORCH_CUDA_ARCH_LIST="10.0"
 export PYTHONNOUSERSITE=1
 export VLLM_FLOAT32_MATMUL_PRECISION=high
-GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.90}"
+# Reduced cache headroom for source validation; model weights need ~137 GiB/GPU.
+GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.85}"
 
 { set +x; } 2>/dev/null
 VLLM_CMD=(

@@ -962,7 +962,7 @@ dsv4-fp4-b200-vllm:
       - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 1024 }
 
 dsv4-fp4-b200-vllm-agentic-mtp:
-  image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-426e59f
+  image: cquil11/vllm-cache-sources@sha256:230bee1800e94c1a18c3053d1b5cee7ffd5b73f776a8e9f9f87f30d8df5c629e
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: cluster:b200-nscale
@@ -971,12 +971,14 @@ dsv4-fp4-b200-vllm-agentic-mtp:
   multinode: false
   scenarios:
     agentic-coding:
-    - dram-utilization: 0.95
+    - dram-utilization: 0.30
+      search-space:
+      - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, spec-decoding: mtp, conc-list: [8] }
+    - search-space:
+      - { tp: 8, kv-offloading: nvme, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [14] }
+    - dram-utilization: 0.03
       search-space:
-      # TP8 SimpleCPU + MTP
-      - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [1, 4, 8, 14] }
-      # DEP8 SimpleCPU + MTP
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [32, 64, 96, 128, 160, 196], router: { name: vllm-router, version: "0.1.14" } }
+      - { tp: 8, kv-offloading: [dram, nvme], kv-offload-backend: { name: vllm-native }, spec-decoding: mtp, conc-list: [14] }
 
 dsv4-fp4-b200-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066

@@ -395,7 +395,7 @@ def agentic_dram_offload_gb(
       budgeted separately if it ever gains its own pool).
     """
     kv_offloading = benchmark.get(Fields.KV_OFFLOADING.value, "none")
-    if kv_offloading != "dram":
+    if kv_offloading not in ("dram", ["dram", "nvme"]):
         return 0
 
     available_mib = min(
@@ -429,13 +429,14 @@ def agentic_dram_offload_gb(
 
 
 def agentic_kv_offload_suffix(
-    kv_offloading: str,
+    kv_offloading: str | list[str],
     kv_offload_backend: dict | None,
 ) -> str:
     """Return a compact exp-name suffix for agentic KV offload settings."""
     if kv_offloading == "none":
         return "kvnone"
-    return f"kv{kv_offloading}-{kv_offload_backend['name']}"
+    mode = "+".join(kv_offloading) if isinstance(kv_offloading, list) else kv_offloading
+    return f"kv{mode}-{kv_offload_backend['name']}"
 
 
 def multinode_agentic_exp_name(
@@ -941,7 +942,9 @@ def _agentic_entries(
                 + (f"_spec-{spec_decoding}" if spec_decoding != "none" else "")
             )
         entry.update({
-            Fields.KV_OFFLOADING.value: kv_offloading,
+            Fields.KV_OFFLOADING.value: (
+                "+".join(kv_offloading) if isinstance(kv_offloading, list) else kv_offloading
+            ),
             Fields.TOTAL_CPU_DRAM_GB.value: total_cpu_dram_gb,
             Fields.DURATION.value: DEFAULT_AGENTIC_DURATION_SECONDS,
             Fields.EXP_NAME.value: exp_name,

@@ -14,6 +14,9 @@
 
 CLUSTER_LABEL_PREFIX = "cluster:"
 DEFAULT_AGENTIC_DURATION_SECONDS = 3600
+KVOffloadingConfig = Union[
+    Literal["none", "dram", "nvme"], List[Literal["dram", "nvme"]]
+]
 
 """
     The below class defines the field names expected to be present in the JSON entries
@@ -328,7 +331,7 @@ class SingleNodeAgenticMatrixEntry(BaseModel):
         default="none", alias=Fields.SPEC_DECODING.value
     )
     conc: int
-    kv_offloading: Literal["none", "dram"] = Field(
+    kv_offloading: Literal["none", "dram", "nvme", "dram+nvme"] = Field(
         alias=Fields.KV_OFFLOADING.value
     )
     kv_offload_backend: Optional[KVOffloadBackendMetadata] = Field(
@@ -531,7 +534,10 @@ def _validate_kv_offload_fields(self):
                 f"{Fields.KV_OFFLOADING.value}"
             )
         return self
-    if self.kv_offloading == "none":
+    if isinstance(self.kv_offloading, list):
+        if self.kv_offloading != ["dram", "nvme"]:
+            raise ValueError("The only supported tier list is ['dram', 'nvme']")
+    elif self.kv_offloading == "none":
         if backend is not None:
             raise ValueError(
                 f"{Fields.KV_OFFLOAD_BACKEND.value} can only be set when "
@@ -662,7 +668,7 @@ class AgenticCodingSearchSpaceEntry(BaseModel):
     decode: Optional[WorkerConfig] = None
     num_nodes: Optional[int] = Field(
         default=None, alias=Fields.NUM_NODES.value, gt=0, strict=True)
-    kv_offloading: Optional[Literal["none", "dram"]] = Field(
+    kv_offloading: Optional[KVOffloadingConfig] = Field(
         default=None, alias=Fields.KV_OFFLOADING.value
     )
     kv_offload_backend: Optional[KVOffloadBackendMetadata] = Field(
@@ -710,6 +716,8 @@ def validate_topology_fields(self):
                 )
             _validate_tp_context_topology(self)
         if has_aggregate_worker or has_complete_multinode:
+            if self.kv_offloading in ("nvme", ["dram", "nvme"]):
+                raise ValueError("NVMe offloading currently requires a single-node entry")
             explicitly_single_node_fields = {
                 "pp",
                 "dcp_size",
@@ -744,7 +752,7 @@ class AgenticCodingConfig(BaseModel):
     @model_validator(mode='after')
     def validate_dram_offload_capacity(self):
         for entry in self.search_space:
-            if entry.kv_offloading != "dram":
+            if entry.kv_offloading not in ("dram", ["dram", "nvme"]):
                 continue
             if self.dram_utilization is None:
                 raise ValueError(

@@ -7021,3 +7021,50 @@
     - "Refresh the full GB200 Dynamo-vLLM AgentX configuration across the configured aggregate and disaggregated topology points."
     - "Retain aggregate SimpleCPU-offload concurrency 20 and 30, and configure Mooncake host KV storage, NIXL over UCX, explicit thinking mode, and the committed MiniMax-M3 EAGLE3-GQA acceptance target."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2807
+
+- config-keys:
+    - dsv4-fp4-b200-vllm-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Validate vLLM cache-source attribution on the existing DeepSeek-V4-Pro B200 TP8 MTP AgentX recipe: native DRAM c8, Simple NVMe c14, and native DRAM+NVMe c14."
+    - "Use cquil11/vllm-cache-sources:b0dc209c-swa-amd64-20260909, built from the exact upstream-base wheel plus cquil11/vllm#2 at b0dc209c14451e9a4e63f1db148ed492d881b765."
+    - "Reduce GPU memory utilization to 0.85 and the tiered host cache to 64 GB to exercise filesystem promotions; retain canonical AgentX duration and collect AIPerf Prometheus exports."
+  no-evals: true
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2931
+
+- config-keys:
+    - dsv4-fp4-b200-vllm-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Use the r2 Docker Hub image with matching FlashInfer Python, cubin, and CUDA 13 JIT-cache packages at 0.6.18; retain the same three cache-source validation points."
+  no-evals: true
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2931
+
+- config-keys:
+    - dsv4-fp4-b200-vllm-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Set OMP_NUM_THREADS=1 explicitly to preserve the proven TP8 recipe's CPU parallelism; the new runtime otherwise selects 24 startup threads per worker on B200 Nscale."
+  no-evals: true
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2931
+
+- config-keys:
+    - dsv4-fp4-b200-vllm-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Remove the unproven OMP_NUM_THREADS override; keep the runtime default for official cache-source validation."
+  no-evals: true
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2931
+
+- config-keys:
+    - dsv4-fp4-b200-vllm-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  no-evals: true
+  description:
+    - "Use the Python frontend that exports the fork's cache-source metric; the Rust frontend has its own metrics implementation and does not yet consume the source breakdown. Retain the existing engine, attention, TP8/MTP, and offload settings."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2931
@@ -28,6 +28,9 @@ elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" && $MODEL == "deepseek-ai
 elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then
     # Node-local weights are not visible on the runner/login node.
     export MODEL_PATH="/scratch/models/DeepSeek-V4-Pro-NVFP4"
+    if [[ "$FRAMEWORK" == "vllm" ]]; then
+        export MODEL_PATH="/scratch/models/DeepSeek-V4-Pro"
+    fi
-    if [[ "$FRAMEWORK" == "vllm" ]]; then
-        export MODEL_PATH="/scratch/models/DeepSeek-V4-Pro"
-    fi
+    if [[ "$FRAMEWORK" == "vllm" && "${SCENARIO_TYPE:-}" == "agentic-coding" ]]; then
+        export MODEL_PATH="/scratch/models/DeepSeek-V4-Pro"
+    fi
-    if [[ "$FRAMEWORK" == "vllm" ]]; then
-        export MODEL_PATH="/scratch/models/DeepSeek-V4-Pro"
-    fi
+    if [[ "$FRAMEWORK" == "vllm" && "${SCENARIO_TYPE:-}" == "agentic-coding" ]]; then
+        export MODEL_PATH="/scratch/models/DeepSeek-V4-Pro"
+    fi
     export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro"
 elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "bf16" ]]; then
     export MODEL_PATH="/scratch/models/Qwen3.5-397B-A17B"
@@ -531,8 +534,33 @@ else
     export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}"
 
     SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-480}"
-    salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT --gres=gpu:$GPU_COUNT --exclusive --mem=0 --time="$SALLOC_TIME_LIMIT" --no-shell --job-name="$RUNNER_NAME"
+    salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT --gres=gpu:$GPU_COUNT --exclusive --mem=0 --time="$SALLOC_TIME_LIMIT" --no-shell --job-name="$RUNNER_NAME" || exit 1
     JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1)
+    [[ "$JOB_ID" =~ ^[0-9]+$ ]] || { echo "Could not resolve Slurm allocation" >&2; exit 1; }
+
+    NVME_HOST_DIR=""
+    cleanup_offload_cache() {
+        local rc=$?
+        if [[ -n "$NVME_HOST_DIR" ]]; then
+            timeout --kill-after=10s 60s srun --jobid="$JOB_ID" \
+                rm -rf -- "$NVME_HOST_DIR" || {
+                    echo "NVMe cleanup failed: $NVME_HOST_DIR on job $JOB_ID" >&2
+                    rc=1
+                }
+        fi
+        scancel "$JOB_ID" || true
+        exit "$rc"
+    }
+    trap cleanup_offload_cache EXIT
+    trap 'exit 130' INT
+    trap 'exit 143' TERM
+    EXTRA_MOUNTS=""
+    if [[ "${KV_OFFLOADING:-none}" == *nvme* ]]; then
+        NVME_HOST_DIR="/scratch/inferencex-kv-$JOB_ID"
+        srun --jobid="$JOB_ID" mkdir -m 700 "$NVME_HOST_DIR" || exit 1
+        export NVME_OFFLOAD_DIR=/kv-offload
+        EXTRA_MOUNTS=",$NVME_HOST_DIR:$NVME_OFFLOAD_DIR"
+    fi
 
     # Point the bench script at the resolved MODEL_PATH instead of
     # pulling from the HF hub cache. Bench scripts skip `hf download` when
@@ -550,13 +578,13 @@ else
             echo 'Squash file already exists and is valid, skipping import'
         else
             rm -f \"$SQUASH_FILE\"
-            enroot import -o \"$SQUASH_FILE\" docker://$IMAGE
+            enroot import -o \"$SQUASH_FILE\" docker://${IMAGE/@/:}
         fi
-    "
+    " || exit 1
 
     srun --jobid=$JOB_ID \
         --container-image=$SQUASH_FILE \
-        --container-mounts=$GITHUB_WORKSPACE:$CONTAINER_MOUNT_DIR,$MODEL_PATH:$MODEL_PATH,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache \
+        --container-mounts=$GITHUB_WORKSPACE:$CONTAINER_MOUNT_DIR,$MODEL_PATH:$MODEL_PATH,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache$EXTRA_MOUNTS \
         --no-container-mount-home \
         --container-workdir=$CONTAINER_MOUNT_DIR \
         --no-container-entrypoint --export=ALL,PORT=8888,AIPERF_DATASET_MMAP_CACHE_DIR=/aiperf_mmap_cache \