diff --git a/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml b/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml index 3f994440b..8465f5f88 100644 --- a/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml +++ b/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml @@ -167,6 +167,7 @@ strategy_overrides: tp: default: 4 h100: 8 + b200: 2 b300: 2 hardware_overrides: blackwell: @@ -177,6 +178,28 @@ strategy_overrides: - "fp8" - "--max-num-seqs" - "128" + # TP2 on 180 GB B200 with the Engram tables in pinned host DRAM behind + # UVA still leaves ~145 GiB of resident weights per GPU, so the memory + # levers from the InferenceX B200 arm apply + # (SemiAnalysisAI/InferenceX#3216, AgentX c1-c128 green in run + # 35480787586): at the upstream 16384 batched tokens the + # sparse-attention indexer's [batched-tokens, 1M] fp8 logits buffer is + # 32 GiB and graph capture above 512 tokens pushed the KV budget + # negative, so batched tokens are capped at 4096 (8 GiB) and CUDA-graph + # capture stops at 512 tokens; larger decode batches run eagerly. + # FlashInfer's autotune batches max-num-seqs requests through the DSpark + # draft head and picked an invalid MXFP8 tactic below 16 sequences, so + # keep --max-num-seqs at 16 or above. + b200: + extra_args: + - "--engram-config" + - '{"cpu_offload":true}' + - "--max-num-batched-tokens" + - "4096" + - "--max-cudagraph-capture-size" + - "512" + - "--max-num-seqs" + - "256" b300: extra_args: - "--engram-config" @@ -503,8 +526,15 @@ guide: | settings take precedence over the shared Blackwell defaults. For high interactivity, use `--tensor-parallel-size 4`. - These command settings were updated on 2026-09-20; they have not been rerun - on GPUs as part of this recipe update. + B200 (180 GB per GPU) also defaults to **TP2** with Engram CPU offload. Its + exact-GPU override caps `--max-num-batched-tokens` at 4096 and + `--max-cudagraph-capture-size` at 512 and sets `--max-num-seqs 256`, the + settings under which the InferenceX AgentX sweep completed concurrency 1 + through 128 ([InferenceX #3216](https://github.com/SemiAnalysisAI/InferenceX/pull/3216), + run 35480787586). For high interactivity, use `--tensor-parallel-size 4`. + + These command settings were updated on 2026-09-20; the B300 settings have not + been rerun on GPUs as part of this recipe update. ## KV cache offloading