diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh index faf2232726..1d9bff1320 100755 --- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh @@ -240,9 +240,9 @@ VLLM_CMD=( --reasoning-parser kimi_k3 --tool-call-parser kimi_k3 --enable-auto-tool-choice - # FP8 KV cache requires the prefill query quantization flag. MLA prefill - # runs on TRTLLM_RAGGED per the upstream Blackwell override. - --attention-config '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # FP8 KV cache requires prefill query quantization. Use FlashInfer MLA + # prefill, matching the current published upstream Kimi-K3 recipe. + --attention-config '{"mla_prefill_backend":"flashinfer","use_prefill_query_quantization":true}' --speculative-config "$SPEC_CONFIG" --compilation-config "$COMPILATION_CONFIG" --disable-uvicorn-access-log diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f916be18dc..a376a9a16b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1521,7 +1521,7 @@ kimik3-fp4-b300-vllm-agentic-dspark: # its native window is 32k YaRN-stretched to 1M, and the same image/draft/config # reaches AL 4.18 on short-context work. These throughput numbers measure the # system at a prescribed acceptance, not the draft's fitness at 100k+ context. - image: vllm/vllm-openai:kimi-k3 + image: vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:b300-nv @@ -1533,17 +1533,15 @@ kimik3-fp4-b300-vllm-agentic-dspark: agentic-coding: - dram-utilization: 0.63 search-space: - # Mirrors the non-MTP entry's KV arms so the spec-decoding delta is - # readable at equal concurrency, but stops at conc 16 rather than 24: the - # non-MTP bring-up sweep (run 30326393603) showed the GPU-resident arm - # already thrashing at conc >= 16 (prefix cache hit 2.7%, TTFT p50 86-191s) - # because GPU KV holds only ~3.1 max-length requests, so conc 24 costs a - # full job per arm to re-measure that regime. conc 16 still exercises the - # DRAM tier meaningfully (62% external prefix cache hit rate there). - # TP8-only for the same memory reason: a ~1.5 TB MXFP4 checkpoint does not - # fit below 8 GPUs. + # Keep equal resident/offload points through conc 8. The digest-pinned + # bring-up sweep (run 31404943911) confirmed that resident conc 16 + # remained in warmup after 1,770 seconds (96/177 requests), while conc 8 + # entered profiling. The offload conc 16 arm completed warmup in 804 + # seconds and exercised the external KV tier, so retain it as the + # capacity endpoint. + # TP8-only: a ~1.5 TB MXFP4 checkpoint does not fit below 8 GPUs. # TP8 GPU-resident - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 16] } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8] } # TP8 SimpleCPUOffload (host DRAM) - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16] } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 41e7238f8d..7ae8479441 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5784,3 +5784,13 @@ - "Update the B300 DSV4 AgentX MTP image and SimpleCPU sweep through c576." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2536 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark + description: + - "Refresh B300 Kimi K3 DSpark submission with corrected AgentX harness" + - "Use the pinned 2026-08-10 vLLM nightly with upstream Kimi K3 and DSpark support" + - "Match the current published upstream Kimi K3 FP8-KV profile by using flashinfer MLA prefill with prefill query quantization" + - "Keep DSpark K=2 probabilistic throughput on synthetic golden AL 2.51 while the generated EVAL_ONLY row uses real block verification" + - "Cap the GPU-resident search at concurrency 8 after concurrency 16 failed to complete deterministic warmup; retain concurrency 16 for the DRAM-offload capacity tier" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2476