diff --git a/.github/AGENT_OPERATIONS.md b/.github/AGENT_OPERATIONS.md index 84d8e86e6d..92aa00e9e9 100644 --- a/.github/AGENT_OPERATIONS.md +++ b/.github/AGENT_OPERATIONS.md @@ -96,7 +96,7 @@ Multinode disaggregated results add `prefill_gpu_energy_j`, `decode_gpu_energy_j Every power result — valid or invalid, single-node or multinode — carries `power_metric_schema_version`. Version 2 defines each unprefixed `joules_per_*` field as whole-deployment GPU-board energy over the named denominator; role-scoped energy uses the explicit `prefill_*` / `decode_*` keys. Rows without the field predate the whole-deployment switch and their unprefixed joules are not comparable across topologies. -For srt-slurm recipes, `telemetry: {provider: dcgm-power}` enables official energy collection. `runners/launch_gb200-nv.sh`, `runners/launch_gb300-nv.sh`, and `runners/launch_h200-dgxc-slurm.sh` are the source of truth for `POWER_SRT_SLURM_PIN`. CI derives `POWER_PRODUCER_SHA` from the launcher stamp. `utils/test_gb300_power_official_contract.py` exercises launcher routing; the aggregate-power and AgentX power tests validate telemetry, provenance, and lifecycle behavior. These local tests do not prove hardware power collection. Eligible recipe-gated `dynamo-sglang` dcgm-power lanes are validated. +For srt-slurm recipes, `telemetry.enabled: true` with `telemetry.dcgm_exporter` enables official energy collection. The Git submodule pointer at `utils/srt-slurm` is the source of truth for the shared srt-slurm commit, used by both power and non-power NVIDIA lanes. TileRT is the single documented fork exception. CI derives `POWER_PRODUCER_SHA` from the launcher stamp. The aggregate-power and AgentX power tests validate telemetry and provenance. These local tests do not prove hardware power collection. Eligible recipe-gated `dynamo-sglang` dcgm-power lanes are validated. Power audit artifacts are named `power_audit_` and contain `power_validation_.json` for single-node runs or `power_validation__*.json` for multinode runs. They are uploaded even when validation fails. diff --git a/.github/workflows/test-process-result.yml b/.github/workflows/test-process-result.yml index 3be431450e..e9e067e1d8 100644 --- a/.github/workflows/test-process-result.yml +++ b/.github/workflows/test-process-result.yml @@ -15,13 +15,13 @@ on: - 'runners/test_native_collector_barriers.py' - 'runners/test_native_collector_receipts.py' - 'utils/test_native_multinode_power.py' - - 'benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/**/*.yaml' - - 'benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/**/*.yaml' - - 'benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/**/*.yaml' - - 'benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml' - - 'benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml' - - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-*/**/*.yaml' - - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-*/**/*.yaml' + - 'benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/**/*.yaml' + - 'benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/**/*.yaml' + - 'benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/**/*.yaml' + - 'benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml' + - 'benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml' + - 'benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-*/**/*.yaml' + - 'benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-*/**/*.yaml' - 'configs/nvidia-master.yaml' - 'runners/launch_gb200-nv.sh' - 'runners/launch_gb300-nv.sh' diff --git a/.gitmodules b/.gitmodules index 79fbb076aa..f7635a307b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "utils/aiperf"] path = utils/aiperf url = https://github.com/SemiAnalysisAI/aiperf.git +[submodule "utils/srt-slurm"] + path = utils/srt-slurm + url = https://github.com/NVIDIA/srt-slurm.git diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 88a61ccc10..d7161e49eb 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -403,7 +403,7 @@ DeepSeek-V4-Pro-DI: # DeepSeek-V4 built-in MTP: load via the EAGLE spec path with eagle-topk 1 (a single # MTP chain). NOT NEXTN -- the V3/R1 NEXTN loader crashes the dsv4 decode server at # init on the V4 architecture. Mirrors the sglang dsv4 recipes - # (srt-slurm-recipes/sglang/deepseek-v4/*: EAGLE + eagle-topk 1, in-checkpoint draft, + # (srt-slurm-recipes/dsv4/sglang/*: EAGLE + eagle-topk 1, in-checkpoint draft, # no draft-model-path). Consumed by build_server_config only when DECODE_MTP_SIZE > 0 # (spec-decoding: mtp); a no-op for the spec-none base scenario (DECODE_MTP_SIZE=0). # build_server_config appends --speculative-num-steps/--speculative-num-draft-tokens diff --git a/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index 2ecdcb816c..a731508e05 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -1,92 +1,79 @@ -# Registering Recipes from srtslurm +# srt-slurm recipes -For disaggregated multi-node configurations (`dynamo-sglang`, `dynamo-trt`), recipes are stored in the external [srtslurm](https://github.com/NVIDIA/srt-slurm) repository. This doc covers staging those recipes in InferenceX. +**English** | [中文](./RECIPES_zh.md) -## 1. Locate source recipes in srtslurm +InferenceX owns the recipes in this directory. Every NVIDIA srt-slurm launcher uses `setup_srt_slurm()` in [`runners/slurm_utils.sh`](../../../runners/slurm_utils.sh), makes a job-local Git clone of the pinned submodule, and copies this entire tree into `recipes/`. The shared helper records the actual revision in `srt-slurm-sha.txt`; power lanes copy that revision into `power-producer-sha.txt` for result validation. -```bash -# Example: H200 sglang disagg recipes -ls /path/to/srtslurm/recipes/h200/ -# 1k1k/ 8k1k/ -``` +The shared version is the Git submodule pointer at [`utils/srt-slurm`](../../../utils/srt-slurm), currently [v2.2.1](https://github.com/NVIDIA/srt-slurm/releases/tag/v2.2.1) (`984180e5b8755aef85e9995048b5a16cb5336bce`). Update that submodule pointer when upgrading, then run the recipe and integration checks. Do not add model-specific checkout branches to launchers. -## 2. Recipe structure - -Each recipe YAML contains: -- `name`: Recipe identifier -- `model`: Model path/container info -- `resources`: GPU type, prefill/decode node/worker counts -- `backend.sglang_config`: Prefill and decode configuration (tp-size, dp-size, ep-size, dp-attention, etc.) -- `benchmark`: ISL/OSL and concurrency settings - -## 3. Add config to nvidia-master.yaml - -```yaml -dsr1-fp8-h200-dynamo-sglang: - image: lmsysorg/sglang:v0.5.8-cu130-runtime - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: cluster:h200-dgxc - precision: fp8 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - conc-list: [1, 4, 16, 32, 64, 128, 256, 512] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/1k1k/bs128-agg-tp.yaml" - decode: - num-worker: 0 - tp: 8 - ep: 1 - dp-attn: false -``` +InferenceX requires srt-slurm 2.0 or newer and `schema: 2` recipes. Legacy recipe layouts are unsupported; migrate them before adding them to this tree. + +## Directory and filename convention + +Store every recipe at `//-//.yaml`: -## 4. Field mapping (srtslurm → nvidia-master.yaml) - -| srtslurm field | nvidia-master.yaml field | -|----------------|-------------------------| -| `resources.prefill_workers` | `prefill.num-worker` | -| `resources.decode_workers` | `decode.num-worker` | -| `sglang_config.prefill.tp-size` | `prefill.tp` | -| `sglang_config.prefill.ep-size` | `prefill.ep` | -| `sglang_config.prefill.enable-dp-attention` | `prefill.dp-attn` | -| `benchmark.concurrencies` (parsed) | `conc-list` | -| Recipe file path | `additional-settings: CONFIG_FILE=...` | - -## 5. Common patterns - -- **Aggregated (AGG)**: Single node, `num-worker: 1` for prefill, `num-worker: 0` for decode -- **TEP (Tensor-Expert Parallel)**: `dp-attn: false`, `ep: 1` -- **DEP (Data-Expert Parallel)**: `dp-attn: true`, `ep: 8` (typically) -- **Low latency**: More decode workers (e.g., 9), lower concurrencies -- **High throughput**: Fewer decode workers, higher concurrencies - -## 6. Add perf-changelog entry - -```yaml -- config-keys: - - dsr1-fp8-h200-dynamo-sglang - description: - - "Add DSR1 FP8 H200 Dynamo SGLang disaggregated multinode configuration" - - "Image: lmsysorg/sglang:v0.5.8-cu130-runtime" - - "Recipes sourced from srtslurm repo (recipes/h200/)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX +```text +dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml +glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml +qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml ``` -## 7. Validate +- Use the master config's `model-prefix` and `precision` labels. Engines are `sglang`, `vllm`, `trtllm`, and `tilert`; frontend selection remains explicit inside the recipe. Hardware directories use GPU types such as `b200` and `gb300`, rather than cluster names. +- Workloads are `1k1k`, `8k1k`, or `agentx`. Existing bundles spanning several fixed sequence lengths use `fixed-seq-len`; keep their override selectors intact. +- Use lowercase, hyphen-separated filenames beginning with `agg` or `disagg`. Include topology and the settings that distinguish sibling recipes, such as parallelism, batch size, concurrency, MTP, offload, or cache configuration. Avoid dates, numbered latency/throughput labels, and repeating the model or hardware already in the path. +- In topology names, `1p4d` denotes prefill/decode worker counts, not necessarily physical nodes. Role-qualified `p-tp4` and `d-tp8` identify prefill/decode TP; `b` denotes batch size and `c` concurrency. The YAML is authoritative for runtime settings. +- Name override bundles `*-variants.yaml`. Keep distinct sweep entry files separate even when their contents match: recipe paths participate in eval grouping. The Qwen3.5 `*-stp-sweep.yaml` and `*-mtp-sweep.yaml` pair preserves that existing distinction. +- Update `CONFIG_FILE` and `EVAL_CONFIG_FILE` references in active and deprecated master configs, launcher path rules, workflow filters, and local documentation together when moving a file. Preserve upstream source URLs as provenance and leave historical performance-changelog entries unchanged. No aliases for the old layout are provided. + +Shared runtime assets stay under `configs/` beside the model directories; they are not standalone recipes. The four files in `configs/dsv4-moe-load-balancer-configs/` are copied verbatim from NVIDIA/srt-slurm commit `deb1dfd9934398664f92d194169c183e009da83b`, preserving the EPLB initial expert assignments used by 17 DSV4 TRT recipes. `setup_srt_slurm()` stages them into the job checkout's `configs/` directory for the recipes' bind mounts. Keeping a recipe in this tree does not activate it; the master configs determine the benchmark matrix. + +## TileRT exception + +For `FRAMEWORK=tilert`, `setup_srt_slurm()` fetches the SemiAnalysisAI/srt-slurm fork directly at `6bc3f306bdafa1edfb5dded2fcda8f1ccede1bde` into the job checkout. This is the schema-2 TileRT port in [SemiAnalysisAI/srt-slurm#13](https://github.com/SemiAnalysisAI/srt-slurm/pull/13). It is the only alternate checkout; its pin lives in that helper because the TileRT backend and router are absent from the NVIDIA pin. TileRT uses the same schema-2 recipe layout and native post-eval dispatch as NVIDIA. TileRT jobs need network access to the fork at setup time. Remove the fork exception once those features are available upstream. + +## Schema 2 and master configuration + +Recipes use `schema: 2`, `engine`, and `roles`. Each worker role owns its node count, worker count, GPU allocation, environment, and engine arguments. `resources` retains GPU hardware facts. `placement` controls the frontend and benchmark location, `services` describes auxiliary processes, and `dynamo.source` selects the Dynamo package or source revision. + +| Recipe field | `configs/nvidia-master.yaml` field | +|---|---| +| `roles.prefill.workers` | `prefill.num-worker` | +| `roles.decode.workers` | `decode.num-worker` | +| `roles.prefill.args.tp-size` (SGLang) | `prefill.tp` | +| `roles.prefill.args.ep-size` (SGLang) | `prefill.ep` | +| `roles.prefill.args.enable-dp-attention` | `prefill.dp-attn` | +| `benchmark.concurrencies` | `conc-list` | +| Recipe path, optionally with an override selector | `additional-settings: CONFIG_FILE=recipes/...yaml` | + +Keep the recipe and master configuration synchronized. The launcher executes the recipe; the master configuration supplies result labels and scheduling metadata. For aggregate recipes use `roles.agg`; `roles.decode.nodes: colocate` shares prefill nodes and contributes no additional worker nodes to scheduling. + +All referenced recipes must be checked in: srt-slurm 2 ships curated examples instead of the historical `recipes/` archive. The initial migration restores 204 previously external recipes and two still-referenced AgentX recipes from InferenceX history. Master-config paths follow the layout above; existing override selectors are preserved. + +## Migration and validation + +Install the shared pin in an isolated environment, then use its CLI: ```bash +# Verify each supported recipe directory before rewriting it. +srtctl migrate --verify -f benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang +srtctl migrate --in-place -f benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang +# Repeat for the other model/engine directories. +# Use the pinned TileRT fork for glm5.1/tilert/. +python -m pytest utils/matrix_logic/ -q python -m infx.matrix.generate full-sweep \ --config-files configs/nvidia-master.yaml \ - --framework dynamo-sglang + --framework dynamo-sglang dynamo-trt dynamo-vllm --multi-node ``` + +Validate recipes with the exact launcher pin, including all override variants. For a path-only reorganization, compare generated matrices before and after with the path mapping applied; all other fields, including eval selection and node counts, must match. A passing local schema check does not replace the full hardware sweep and evals. + +The initial migration also resolves compatibility issues that `srtctl migrate` cannot fix itself: + +- SGLang Model Gateway recipes use `frontend.type: sglang-router`; in v2.2.1, `sglang` selects a direct worker without a router. +- Duplicate YAML keys retain the value selected by the former PyYAML loader. +- DCGM telemetry uses `collect_interval_ms: 1000` instead of `provider` and `default_frequency`. The collector derives its shutdown budget; an explicit ten-second budget is too short for the current validator. Dedicated discovery-service placement is preserved from the original recipes. The pinned upstream runtime rejects telemetry with dedicated infrastructure nodes; this remains a power compatibility blocker rather than changing the original topology to satisfy validation. H200 custom recipes declare a default concurrency that the launcher replaces before submission. +- DeepSeek-V4 vLLM benchmarks use the supported `custom_tokenizer` loader. Retired `warmup_req_rate: inf` fields are removed; the current upstream client uses its fixed warmup rate of 250 requests per second. +- The power reader accepts both generations of samples CSV while validating utilization values and continuing to compute board energy from watts. +- Post-eval selection uses native `post_eval.command` and `post_eval.passthrough_env` with [`srt_eval.sh`](../srt_eval.sh). TRT AgentX recipes declare their existing Dynamo fork with `dynamo.source.git`; launchers no longer rewrite the srt-slurm source. + +Append a new entry to the physical end of `perf-changelog.yaml` for every recipe or runtime change. Preserve all historical bytes. Validate the PR with `full-sweep-fail-fast`, including evals, before following the repository's review and artifact-reuse merge process. diff --git a/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md b/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md new file mode 100644 index 0000000000..ef3c0ed034 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md @@ -0,0 +1,79 @@ +# srt-slurm 配置 + +[English](./RECIPES.md) | **中文** + +InferenceX 负责维护本目录中的配置。所有 NVIDIA srt-slurm 启动器均调用 [`runners/slurm_utils.sh`](../../../runners/slurm_utils.sh) 中的 `setup_srt_slurm()`，为作业创建固定版本子模块的本地 Git 克隆，并将整个目录复制到 `recipes/`。共享函数将实际提交记录到 `srt-slurm-sha.txt`；功耗测试路径还会将其复制到 `power-producer-sha.txt`，供结果校验使用。 + +统一版本由 [`utils/srt-slurm`](../../../utils/srt-slurm) 的 Git 子模块指针指定，目前为 [v2.2.1](https://github.com/NVIDIA/srt-slurm/releases/tag/v2.2.1)（`984180e5b8755aef85e9995048b5a16cb5336bce`）。升级时更新该子模块指针，然后运行配置和集成检查。不要在启动器中新增按模型选择检出版本的分支。 + +InferenceX 要求 srt-slurm 2.0 或更新版本，且配置必须声明 `schema: 2`。不支持旧版配置结构；加入本目录前必须先完成迁移。 + +## 目录和文件命名规范 + +所有配置统一存放在 `//-//.yaml`： + +```text +dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml +glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml +qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml +``` + +- 使用主配置中的 `model-prefix` 和 `precision` 标签。引擎目录为 `sglang`、`vllm`、`trtllm` 或 `tilert`；前端仍在配置内显式声明。硬件目录使用 `b200`、`gb300` 等 GPU 型号，不使用集群名称。 +- 工作负载目录为 `1k1k`、`8k1k` 或 `agentx`。已有的跨序列长度配置集合放在 `fixed-seq-len` 下，保留其覆盖项选择器。 +- 文件名使用小写字母和连字符，以 `agg` 或 `disagg` 开头。包含拓扑及用于区分同目录配置的关键参数，例如并行方式、批大小、并发数、MTP、卸载或缓存设置。避免日期、带序号的延迟/吞吐量标签，以及重复目录中已有的模型或硬件信息。 +- 拓扑名中的 `1p4d` 表示预填充/解码 worker 数，不一定等于物理节点数。`p-tp4` 和 `d-tp8` 分别标识预填充和解码 TP；`b` 表示批大小，`c` 表示并发数。运行参数以 YAML 为准。 +- 覆盖项集合使用 `*-variants.yaml` 命名。即使内容相同，也保留独立扫描入口：配置路径参与评估分组。Qwen3.5 的 `*-stp-sweep.yaml` 和 `*-mtp-sweep.yaml` 保留了这一既有区别。 +- 移动文件时，同步更新当前及已弃用主配置中的 `CONFIG_FILE`、`EVAL_CONFIG_FILE`，以及启动器路径规则、工作流过滤器和本地文档。保留上游来源 URL，并保持历史性能变更日志不变。不为旧目录结构提供别名。 + +共享运行时资源保留在模型目录旁的 `configs/` 中，不属于独立基准测试配置。`configs/dsv4-moe-load-balancer-configs/` 中的四个文件原样取自 NVIDIA/srt-slurm 提交 `deb1dfd9934398664f92d194169c183e009da83b`，保留了 17 个 DSV4 TRT 配置使用的 EPLB 初始专家分配。`setup_srt_slurm()` 将这些文件复制到作业仓库的 `configs/` 目录，供配置中的绑定挂载使用。将配置文件放入本目录不会启用该配置；实际基准测试矩阵由主配置决定。 + +## TileRT 例外 + +当 `FRAMEWORK=tilert` 时，`setup_srt_slurm()` 直接从 SemiAnalysisAI/srt-slurm 分支仓库获取提交 `6bc3f306bdafa1edfb5dded2fcda8f1ccede1bde`，检出到作业目录。该版本为 [SemiAnalysisAI/srt-slurm#13](https://github.com/SemiAnalysisAI/srt-slurm/pull/13) 中支持 schema 2 的 TileRT 移植。这是唯一的备用检出路径；由于统一的 NVIDIA 版本尚未包含 TileRT 后端和路由器，该例外的固定提交在共享函数中指定。TileRT 使用与 NVIDIA 相同的 schema 2 配置结构和原生评估调度。TileRT 作业在准备阶段需要通过网络访问分支仓库。上游支持这些功能后，应删除此分支仓库例外。 + +## Schema 2 与主配置 + +配置使用 `schema: 2`、`engine` 和 `roles`。每个工作角色集中声明节点数、实例数、GPU 分配、环境变量和引擎参数。`resources` 保留 GPU 硬件信息，`placement` 控制前端和基准测试客户端的位置，`services` 描述辅助进程，`dynamo.source` 指定 Dynamo 软件包或源码提交。 + +| 配置字段 | `configs/nvidia-master.yaml` 字段 | +|---|---| +| `roles.prefill.workers` | `prefill.num-worker` | +| `roles.decode.workers` | `decode.num-worker` | +| `roles.prefill.args.tp-size`（SGLang） | `prefill.tp` | +| `roles.prefill.args.ep-size`（SGLang） | `prefill.ep` | +| `roles.prefill.args.enable-dp-attention` | `prefill.dp-attn` | +| `benchmark.concurrencies` | `conc-list` | +| 配置路径，可附带覆盖项选择器 | `additional-settings: CONFIG_FILE=recipes/...yaml` | + +配置文件和主配置必须同步更新。启动器执行配置文件；主配置提供结果标签和调度元数据。聚合式配置使用 `roles.agg`；`roles.decode.nodes: colocate` 表示解码角色与预填充角色共享节点，不增加调度所需的工作节点数。 + +所有被引用的配置都必须纳入版本控制：srt-slurm 2 提供精选示例，不再携带历史 `recipes/` 目录。本次迁移补齐了 204 个此前依赖外部仓库的配置，并从 InferenceX 历史记录恢复了两个仍被引用的 AgentX 配置。主配置路径遵循上述目录结构，原有覆盖项选择器保持不变。 + +## 迁移与验证 + +在隔离环境中安装统一版本，然后使用其 CLI： + +```bash +# 重写前先验证每个受支持的配置目录。 +srtctl migrate --verify -f benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang +srtctl migrate --in-place -f benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang +# 对其他模型/引擎目录重复执行。 +# 迁移 glm5.1/tilert/ 时，使用固定提交的 TileRT 分支仓库。 +python -m pytest utils/matrix_logic/ -q +python -m infx.matrix.generate full-sweep \ + --config-files configs/nvidia-master.yaml \ + --framework dynamo-sglang dynamo-trt dynamo-vllm --multi-node +``` + +使用启动器指定的确切提交验证配置，包括全部覆盖变体。仅调整路径时，应按路径映射比较变更前后的生成矩阵；其他字段（包括评估选择和节点数）必须完全一致。本地配置校验通过不能替代完整硬件扫描和准确性评估。 + +本次迁移还修复了 `srtctl migrate` 无法自动处理的兼容性问题： + +- SGLang Model Gateway 配置使用 `frontend.type: sglang-router`；在 v2.2.1 中，`sglang` 表示不经过路由器的独立工作进程。 +- 对重复的 YAML 键，保留原 PyYAML 加载器实际采用的值。 +- DCGM 遥测使用 `collect_interval_ms: 1000`，替代 `provider` 和 `default_frequency`。采集器自动推导退出等待时间；原先显式设置的十秒不满足当前校验要求。保留原配置中服务发现进程的专用节点部署方式。固定的上游版本不支持在专用基础设施节点上启用遥测；该功耗兼容性问题仍待解决，不通过改变原有拓扑来绕过校验。H200 自定义配置声明默认并发数，提交前由启动器替换。 +- DeepSeek-V4 vLLM 基准测试使用受支持的 `custom_tokenizer` 加载器。删除已废弃的 `warmup_req_rate: inf` 字段；当前上游客户端的预热速率固定为每秒 250 个请求。 +- 功耗读取器兼容两代 samples CSV，校验利用率字段，并继续根据瓦特数计算 GPU 板级能耗。 +- 评估选择通过原生 `post_eval.command` 和 `post_eval.passthrough_env` 调用 [`srt_eval.sh`](../srt_eval.sh)。TRT AgentX 配置通过 `dynamo.source.git` 声明原有的 Dynamo 分支仓库，启动器不再改写 srt-slurm 源码。 + +每次修改配置或运行时，都必须在 `perf-changelog.yaml` 的物理末尾追加新条目，保留全部历史内容及空白。合并前使用 `full-sweep-fail-fast` 验证 PR（包括评估），再按仓库规定完成审查及产物复用合并流程。 diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-moe-load-balancer-configs/moe_load_balancer_ctx_ep4_384.yaml b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-moe-load-balancer-configs/moe_load_balancer_ctx_ep4_384.yaml new file mode 100644 index 0000000000..293e803ea2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-moe-load-balancer-configs/moe_load_balancer_ctx_ep4_384.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [38, 95, 50, 47, 25, 15, 26, 53, 31, 9, 41, 71, 19, 4, 143, 108, 85, 44, 6, 8, 127, 66, 2, 33, 63, 17, 70, 112, 79, 96, 109, 21, 159, 186, 90, 114, 147, 169, 155, 136, 81, 103, 120, 234, 256, 37, 107, 223, 172, 175, 217, 238, 243, 206, 209, 174, 220, 271, 261, 307, 213, 273, 247, 258, 268, 231, 277, 255, 227, 242, 309, 280, 371, 299, 303, 305, 351, 313, 320, 329, 291, 244, 374, 340, 364, 276, 379, 228, 342, 250, 322, 368, 356, 373, 354, 380, 57, 51, 1, 35, 11, 52, 34, 7, 27, 56, 0, 76, 72, 74, 100, 163, 80, 102, 55, 97, 84, 59, 54, 78, 88, 134, 105, 142, 106, 92, 40, 62, 204, 222, 126, 111, 179, 158, 32, 173, 178, 118, 160, 210, 249, 282, 196, 190, 212, 157, 236, 191, 193, 208, 287, 253, 230, 199, 153, 165, 235, 293, 274, 286, 267, 200, 266, 265, 297, 224, 298, 285, 264, 296, 301, 300, 375, 319, 314, 284, 376, 292, 281, 383, 270, 336, 345, 316, 365, 321, 331, 348, 346, 370, 382, 355, 259, 339, 3, 10, 12, 18, 5, 28, 104, 20, 73, 117, 46, 87, 39, 94, 22, 99, 75, 86, 29, 119, 48, 42, 83, 58, 145, 122, 131, 141, 162, 36, 139, 113, 101, 116, 129, 207, 152, 154, 49, 128, 170, 167, 245, 146, 125, 197, 215, 194, 260, 185, 263, 216, 214, 161, 312, 219, 182, 202, 205, 254, 188, 252, 203, 240, 251, 272, 257, 248, 315, 294, 237, 317, 232, 311, 330, 343, 288, 341, 325, 302, 360, 226, 361, 349, 338, 352, 358, 357, 366, 344, 337, 372, 381, 362, 283, 324, 93, 164, 14, 13, 24, 61, 23, 16, 30, 211, 123, 166, 68, 43, 98, 45, 110, 89, 64, 144, 67, 77, 69, 121, 132, 130, 133, 138, 137, 65, 150, 115, 221, 183, 156, 91, 82, 168, 124, 148, 184, 181, 176, 140, 60, 149, 192, 198, 135, 151, 177, 171, 195, 187, 229, 201, 180, 304, 239, 308, 233, 353, 218, 334, 359, 347, 323, 189, 278, 269, 290, 241, 295, 246, 327, 275, 289, 225, 318, 350, 262, 335, 310, 279, 369, 367, 328, 306, 326, 333, 332, 363, 377, 378] + 1: [71, 41, 6, 3, 2, 51, 5, 61, 8, 109, 7, 23, 33, 189, 34, 45, 70, 66, 40, 104, 84, 31, 75, 44, 87, 85, 215, 123, 88, 36, 32, 67, 217, 133, 278, 184, 158, 204, 101, 94, 181, 115, 155, 127, 122, 185, 180, 209, 230, 198, 200, 168, 196, 177, 194, 273, 175, 228, 249, 195, 233, 235, 240, 254, 220, 244, 207, 205, 277, 236, 270, 324, 334, 268, 289, 313, 321, 279, 285, 316, 297, 343, 370, 362, 295, 261, 315, 284, 348, 307, 372, 323, 352, 345, 383, 381, 47, 13, 81, 22, 4, 83, 10, 80, 77, 42, 21, 60, 43, 56, 76, 89, 68, 57, 37, 113, 24, 28, 131, 146, 82, 93, 53, 74, 197, 121, 110, 154, 107, 125, 52, 135, 142, 174, 160, 105, 86, 99, 137, 124, 141, 172, 161, 227, 216, 186, 193, 192, 145, 260, 257, 201, 169, 301, 234, 247, 206, 237, 332, 165, 190, 253, 255, 294, 276, 241, 299, 203, 310, 263, 347, 287, 338, 300, 320, 357, 304, 319, 258, 280, 346, 358, 364, 265, 349, 356, 341, 354, 373, 380, 377, 368, 49, 360, 9, 17, 62, 30, 1, 38, 35, 50, 55, 48, 19, 143, 100, 117, 138, 72, 54, 18, 150, 106, 97, 98, 20, 151, 91, 112, 166, 136, 126, 229, 153, 114, 64, 15, 134, 92, 152, 78, 170, 132, 243, 191, 156, 245, 157, 250, 148, 162, 224, 210, 238, 187, 262, 208, 183, 159, 214, 242, 223, 226, 264, 283, 259, 330, 275, 219, 288, 318, 256, 353, 337, 351, 252, 371, 290, 314, 359, 350, 361, 282, 286, 293, 291, 382, 342, 340, 355, 335, 366, 374, 367, 379, 375, 363, 178, 303, 0, 11, 26, 12, 90, 27, 58, 144, 59, 120, 29, 46, 25, 118, 102, 173, 69, 119, 65, 79, 96, 39, 139, 108, 16, 103, 116, 14, 63, 73, 95, 163, 167, 179, 202, 246, 130, 188, 149, 251, 239, 140, 171, 129, 111, 221, 211, 248, 274, 128, 182, 147, 164, 176, 213, 212, 225, 218, 267, 231, 199, 365, 308, 266, 281, 271, 232, 296, 272, 302, 322, 317, 269, 222, 325, 309, 331, 311, 326, 292, 298, 305, 327, 339, 329, 378, 328, 344, 333, 306, 369, 312, 336, 376] + 2: [79, 33, 43, 22, 1, 76, 52, 14, 16, 40, 12, 20, 2, 0, 102, 44, 60, 34, 24, 83, 128, 50, 136, 78, 99, 54, 73, 143, 120, 122, 118, 107, 85, 88, 172, 137, 212, 153, 152, 138, 97, 178, 206, 163, 279, 188, 184, 146, 213, 174, 237, 113, 125, 242, 233, 285, 183, 284, 201, 259, 243, 229, 241, 264, 236, 234, 323, 203, 303, 275, 367, 249, 232, 281, 258, 338, 289, 265, 286, 305, 301, 345, 326, 378, 322, 380, 371, 347, 331, 336, 344, 315, 271, 310, 300, 375, 86, 39, 6, 28, 53, 35, 127, 46, 66, 94, 89, 109, 25, 30, 13, 47, 71, 90, 96, 27, 72, 9, 169, 67, 38, 131, 141, 92, 108, 110, 26, 100, 68, 139, 167, 150, 223, 158, 80, 202, 154, 185, 155, 246, 288, 200, 299, 124, 221, 191, 176, 215, 151, 69, 132, 210, 224, 280, 180, 220, 225, 317, 298, 319, 226, 197, 291, 340, 283, 294, 307, 218, 278, 346, 297, 261, 320, 235, 270, 357, 329, 348, 316, 293, 370, 360, 342, 376, 377, 328, 312, 335, 358, 372, 365, 379, 133, 296, 10, 87, 5, 36, 55, 42, 29, 11, 82, 106, 45, 41, 126, 74, 48, 64, 18, 119, 84, 121, 58, 21, 17, 192, 63, 101, 162, 81, 267, 61, 181, 98, 240, 222, 214, 166, 140, 145, 111, 129, 130, 194, 177, 269, 211, 175, 205, 247, 148, 302, 144, 216, 187, 251, 238, 164, 230, 263, 156, 195, 290, 250, 182, 217, 186, 273, 274, 333, 308, 373, 276, 268, 228, 311, 254, 350, 245, 309, 266, 256, 318, 349, 313, 327, 231, 364, 339, 381, 295, 314, 374, 366, 341, 383, 199, 123, 4, 77, 8, 7, 115, 15, 51, 93, 37, 23, 65, 70, 49, 3, 31, 56, 134, 104, 32, 19, 142, 114, 105, 75, 62, 165, 103, 161, 168, 149, 59, 117, 57, 112, 135, 171, 170, 160, 193, 95, 219, 91, 209, 190, 287, 173, 157, 196, 255, 147, 189, 207, 159, 116, 253, 204, 179, 198, 282, 227, 272, 257, 244, 277, 292, 208, 239, 324, 262, 252, 334, 248, 343, 351, 306, 304, 352, 321, 356, 325, 368, 382, 332, 337, 260, 361, 330, 353, 354, 369, 363, 359, 362, 355] + 3: [310, 31, 94, 115, 250, 234, 374, 298, 267, 215, 186, 178, 192, 111, 189, 255, 281, 341, 337, 237, 81, 108, 347, 145, 240, 181, 317, 150, 93, 218, 332, 19, 28, 152, 361, 333, 166, 48, 7, 98, 283, 91, 251, 14, 211, 68, 144, 174, 142, 129, 306, 30, 57, 101, 78, 140, 300, 327, 20, 83, 336, 233, 126, 254, 82, 214, 228, 88, 136, 256, 312, 320, 303, 64, 72, 23, 275, 22, 124, 307, 382, 271, 311, 288, 59, 167, 285, 41, 96, 259, 353, 355, 222, 244, 9, 188, 34, 131, 66, 231, 296, 180, 348, 314, 151, 242, 125, 24, 318, 169, 324, 219, 197, 26, 130, 173, 122, 322, 356, 375, 269, 342, 376, 17, 236, 331, 297, 6, 161, 168, 95, 351, 62, 294, 184, 42, 264, 109, 65, 252, 323, 87, 159, 373, 85, 113, 177, 5, 143, 195, 309, 345, 58, 112, 193, 279, 183, 258, 165, 227, 206, 172, 274, 148, 369, 284, 325, 55, 154, 338, 191, 368, 238, 352, 360, 276, 349, 262, 175, 127, 137, 107, 2, 33, 153, 203, 89, 12, 212, 185, 47, 209, 260, 377, 18, 302, 35, 304, 97, 182, 367, 0, 210, 179, 282, 220, 330, 170, 13, 248, 90, 326, 230, 119, 221, 277, 80, 245, 295, 201, 103, 235, 121, 315, 157, 354, 198, 141, 155, 102, 226, 118, 49, 202, 200, 346, 79, 69, 249, 133, 247, 60, 56, 340, 204, 305, 70, 362, 268, 138, 4, 50, 246, 289, 343, 381, 217, 253, 194, 61, 106, 67, 190, 84, 370, 187, 104, 110, 1, 313, 299, 290, 116, 278, 334, 216, 286, 366, 99, 357, 132, 53, 257, 329, 359, 241, 105, 117, 163, 40, 319, 51, 208, 27, 120, 365, 293, 16, 339, 380, 92, 139, 291, 158, 135, 21, 134, 223, 280, 378, 76, 272, 162, 160, 71, 328, 37, 10, 128, 25, 86, 176, 308, 321, 379, 213, 164, 207, 114, 270, 38, 36, 229, 74, 273, 364, 100, 73, 52, 3, 383, 171, 63, 224, 46, 8, 15, 75, 199, 11, 123, 149, 239, 32, 344, 225, 363, 358, 146, 77, 196, 43, 243, 372, 371, 232, 147, 45, 316, 54, 29, 265, 44, 39, 156, 287, 266, 301, 292, 205, 261, 335, 350, 263] + 4: [172, 18, 264, 302, 219, 205, 22, 62, 29, 105, 115, 362, 101, 91, 149, 253, 272, 226, 36, 284, 215, 103, 320, 250, 69, 262, 312, 23, 48, 167, 328, 26, 374, 276, 271, 142, 248, 348, 157, 59, 191, 98, 182, 51, 338, 33, 129, 109, 97, 79, 168, 42, 323, 65, 17, 74, 85, 108, 75, 10, 335, 216, 30, 270, 281, 122, 300, 173, 242, 256, 201, 80, 291, 334, 343, 9, 128, 372, 211, 373, 313, 368, 224, 336, 126, 305, 132, 234, 301, 100, 364, 285, 83, 104, 247, 299, 12, 230, 257, 27, 252, 353, 116, 160, 159, 117, 24, 243, 309, 355, 50, 214, 199, 113, 25, 370, 213, 326, 379, 210, 307, 86, 135, 61, 207, 194, 361, 64, 38, 96, 221, 288, 371, 47, 196, 204, 381, 231, 310, 46, 251, 15, 185, 354, 60, 333, 92, 187, 267, 295, 133, 63, 290, 198, 174, 147, 325, 94, 332, 227, 35, 28, 238, 158, 337, 127, 140, 308, 342, 45, 282, 352, 255, 297, 139, 183, 43, 349, 19, 16, 268, 192, 171, 20, 131, 32, 40, 314, 146, 58, 222, 351, 283, 82, 273, 369, 347, 244, 169, 189, 346, 356, 37, 163, 93, 202, 258, 155, 0, 165, 145, 70, 114, 344, 130, 357, 322, 340, 3, 162, 153, 166, 327, 66, 274, 197, 71, 263, 73, 84, 236, 112, 179, 88, 358, 55, 176, 76, 318, 217, 78, 110, 260, 241, 315, 365, 229, 339, 141, 4, 181, 56, 121, 203, 154, 178, 375, 378, 184, 2, 1, 175, 95, 341, 287, 49, 138, 294, 14, 266, 31, 316, 123, 363, 383, 180, 81, 292, 319, 311, 195, 89, 5, 90, 259, 120, 156, 360, 254, 13, 275, 188, 44, 228, 239, 54, 235, 293, 206, 34, 265, 161, 77, 136, 72, 324, 345, 246, 57, 111, 240, 212, 359, 237, 209, 152, 21, 376, 278, 150, 68, 148, 6, 7, 330, 124, 99, 137, 143, 53, 233, 67, 164, 377, 87, 329, 11, 366, 52, 280, 106, 269, 296, 232, 220, 151, 125, 289, 186, 170, 177, 303, 190, 41, 200, 286, 331, 350, 223, 118, 321, 298, 249, 218, 306, 261, 8, 208, 382, 119, 102, 380, 107, 144, 225, 279, 134, 367, 193, 304, 39, 245, 317, 277] + 5: [214, 133, 193, 178, 131, 67, 46, 242, 69, 161, 35, 41, 299, 365, 188, 209, 361, 45, 4, 11, 379, 314, 66, 330, 149, 47, 284, 257, 244, 168, 308, 117, 200, 38, 106, 2, 266, 378, 146, 90, 62, 323, 225, 121, 16, 152, 186, 343, 22, 277, 30, 377, 24, 143, 9, 337, 380, 344, 5, 359, 219, 222, 123, 31, 320, 335, 32, 169, 172, 158, 367, 185, 294, 3, 360, 201, 54, 10, 349, 70, 307, 354, 110, 28, 50, 205, 98, 270, 51, 75, 348, 23, 321, 223, 63, 177, 165, 274, 276, 352, 287, 249, 217, 129, 190, 346, 120, 301, 351, 280, 61, 243, 21, 293, 147, 333, 291, 198, 332, 124, 232, 305, 59, 65, 298, 40, 206, 261, 347, 211, 184, 238, 64, 366, 60, 334, 227, 93, 52, 207, 36, 42, 364, 180, 82, 103, 202, 135, 286, 164, 282, 92, 183, 283, 220, 89, 88, 94, 236, 15, 187, 71, 107, 251, 29, 44, 127, 373, 99, 345, 248, 336, 234, 350, 155, 278, 26, 302, 126, 57, 229, 109, 85, 116, 362, 150, 271, 295, 275, 111, 300, 318, 128, 230, 372, 304, 182, 262, 81, 265, 383, 241, 309, 199, 231, 108, 137, 181, 159, 154, 194, 324, 306, 114, 14, 329, 339, 357, 338, 78, 132, 279, 80, 355, 8, 144, 252, 134, 259, 381, 218, 290, 48, 136, 34, 53, 216, 105, 0, 119, 171, 166, 153, 233, 374, 55, 115, 20, 122, 228, 157, 56, 140, 341, 255, 325, 273, 316, 95, 224, 145, 6, 195, 104, 353, 382, 43, 18, 83, 73, 91, 268, 174, 162, 33, 375, 311, 97, 170, 281, 312, 141, 368, 226, 285, 125, 142, 196, 269, 296, 49, 246, 17, 138, 213, 74, 151, 79, 250, 156, 130, 258, 239, 163, 221, 342, 13, 1, 173, 191, 118, 27, 245, 319, 260, 264, 167, 19, 256, 39, 112, 84, 370, 327, 197, 215, 7, 272, 113, 204, 254, 77, 210, 179, 289, 58, 369, 76, 100, 235, 192, 303, 317, 322, 356, 253, 267, 212, 203, 176, 208, 189, 358, 297, 12, 315, 292, 160, 340, 139, 240, 328, 175, 148, 102, 237, 101, 313, 331, 288, 72, 371, 86, 68, 96, 310, 326, 247, 363, 87, 25, 37, 376, 263] + 6: [97, 299, 34, 159, 26, 197, 319, 269, 349, 113, 79, 118, 196, 300, 222, 236, 143, 192, 67, 83, 14, 17, 133, 244, 361, 165, 324, 266, 326, 290, 364, 158, 217, 357, 230, 82, 8, 234, 184, 362, 84, 284, 330, 153, 343, 202, 280, 381, 177, 295, 342, 146, 30, 11, 57, 18, 353, 61, 378, 328, 5, 262, 20, 213, 60, 114, 238, 240, 383, 4, 16, 52, 329, 71, 163, 107, 13, 130, 382, 95, 92, 140, 152, 369, 37, 70, 315, 72, 200, 356, 21, 161, 327, 93, 257, 323, 142, 259, 173, 282, 337, 359, 352, 220, 122, 336, 366, 216, 338, 41, 358, 379, 10, 102, 54, 39, 272, 287, 205, 116, 371, 195, 48, 306, 19, 321, 176, 144, 270, 370, 211, 252, 232, 85, 293, 87, 285, 298, 128, 348, 45, 296, 334, 203, 109, 275, 51, 110, 344, 154, 175, 308, 148, 178, 235, 320, 254, 126, 360, 58, 242, 132, 15, 372, 157, 258, 190, 55, 127, 261, 265, 335, 331, 237, 224, 297, 182, 214, 288, 186, 276, 25, 65, 106, 115, 98, 40, 167, 139, 53, 172, 191, 274, 246, 289, 239, 322, 281, 0, 253, 301, 137, 166, 80, 23, 350, 286, 46, 136, 171, 251, 345, 304, 76, 100, 256, 316, 367, 49, 47, 291, 221, 27, 339, 241, 162, 33, 117, 215, 278, 249, 78, 226, 307, 376, 271, 138, 69, 181, 310, 63, 32, 347, 111, 243, 50, 108, 210, 6, 193, 9, 218, 187, 123, 12, 283, 279, 198, 22, 129, 68, 223, 75, 77, 164, 81, 131, 86, 43, 346, 2, 56, 24, 292, 373, 227, 179, 255, 201, 7, 168, 89, 121, 355, 96, 303, 90, 124, 305, 44, 267, 260, 112, 375, 160, 174, 264, 151, 377, 311, 62, 103, 318, 169, 380, 155, 125, 149, 199, 120, 368, 119, 64, 206, 312, 248, 309, 277, 219, 231, 250, 141, 188, 268, 302, 73, 233, 341, 228, 59, 207, 332, 225, 66, 229, 88, 105, 208, 42, 101, 74, 99, 189, 36, 209, 263, 29, 313, 35, 31, 91, 94, 145, 365, 104, 185, 333, 183, 340, 354, 212, 245, 28, 180, 135, 314, 170, 134, 294, 325, 156, 351, 194, 38, 204, 374, 317, 247, 273, 3, 147, 150, 363, 1] + 7: [382, 113, 133, 238, 284, 331, 69, 0, 239, 11, 59, 9, 160, 190, 21, 336, 121, 302, 359, 198, 299, 12, 18, 268, 99, 247, 182, 270, 300, 203, 152, 166, 376, 70, 104, 38, 61, 132, 379, 373, 1, 196, 313, 237, 261, 167, 271, 372, 230, 235, 143, 318, 229, 227, 128, 31, 14, 374, 367, 5, 149, 266, 159, 179, 185, 255, 171, 162, 241, 234, 283, 106, 169, 35, 52, 364, 50, 26, 338, 164, 330, 127, 120, 346, 66, 175, 366, 151, 98, 222, 371, 215, 192, 375, 37, 273, 27, 49, 170, 358, 188, 72, 46, 294, 15, 354, 119, 64, 184, 301, 337, 315, 174, 228, 40, 56, 63, 33, 303, 28, 262, 368, 334, 129, 286, 218, 309, 25, 165, 140, 224, 253, 304, 91, 287, 312, 125, 122, 306, 290, 112, 344, 242, 142, 285, 278, 342, 195, 78, 260, 343, 20, 32, 267, 93, 254, 114, 172, 322, 316, 276, 90, 212, 47, 194, 139, 89, 233, 124, 34, 84, 220, 250, 281, 263, 4, 219, 307, 199, 54, 180, 248, 361, 102, 357, 365, 240, 178, 207, 2, 111, 60, 292, 329, 258, 48, 213, 216, 305, 333, 71, 321, 19, 126, 74, 80, 30, 360, 311, 154, 295, 36, 200, 76, 153, 347, 209, 41, 217, 136, 319, 24, 317, 138, 352, 29, 73, 370, 118, 275, 193, 280, 288, 277, 181, 383, 323, 353, 7, 202, 328, 380, 339, 349, 256, 362, 265, 272, 206, 55, 297, 39, 204, 369, 75, 44, 208, 131, 308, 43, 141, 53, 147, 325, 137, 134, 45, 65, 259, 96, 116, 177, 92, 156, 231, 115, 211, 22, 189, 341, 381, 245, 6, 236, 201, 348, 3, 157, 163, 274, 378, 356, 187, 282, 51, 100, 13, 155, 335, 130, 82, 326, 197, 355, 221, 150, 101, 23, 314, 340, 296, 77, 264, 279, 94, 291, 146, 289, 148, 107, 110, 324, 176, 320, 108, 168, 83, 16, 123, 183, 205, 85, 161, 244, 332, 105, 8, 327, 86, 293, 298, 214, 246, 226, 62, 191, 103, 158, 310, 17, 144, 252, 249, 95, 117, 186, 109, 10, 269, 42, 173, 210, 251, 351, 350, 79, 67, 81, 377, 87, 232, 58, 257, 225, 68, 363, 57, 135, 223, 97, 88, 243, 145, 345] + 8: [290, 17, 315, 35, 371, 367, 227, 351, 171, 335, 225, 131, 40, 185, 376, 337, 221, 352, 278, 275, 282, 329, 330, 263, 301, 12, 238, 92, 114, 245, 243, 271, 115, 137, 179, 224, 284, 287, 205, 308, 110, 177, 359, 383, 248, 264, 251, 162, 366, 222, 345, 332, 299, 147, 43, 239, 183, 109, 72, 55, 73, 2, 274, 23, 124, 283, 184, 253, 199, 328, 146, 24, 250, 188, 178, 244, 134, 192, 379, 59, 267, 145, 357, 44, 163, 97, 78, 54, 136, 270, 15, 216, 113, 214, 102, 26, 88, 231, 62, 336, 354, 249, 103, 158, 381, 27, 90, 52, 38, 309, 80, 82, 150, 8, 33, 129, 215, 10, 257, 241, 65, 194, 344, 360, 276, 6, 3, 219, 117, 382, 187, 125, 132, 280, 259, 81, 196, 209, 191, 144, 42, 247, 14, 139, 213, 237, 180, 198, 327, 343, 22, 1, 189, 272, 57, 197, 334, 260, 311, 133, 190, 324, 95, 340, 66, 300, 159, 63, 39, 348, 365, 338, 165, 25, 230, 29, 320, 30, 310, 305, 212, 46, 108, 151, 116, 5, 254, 206, 234, 314, 120, 369, 362, 312, 7, 353, 156, 152, 50, 18, 84, 61, 148, 223, 195, 94, 240, 279, 255, 153, 70, 289, 372, 228, 296, 377, 355, 37, 31, 293, 356, 122, 45, 319, 307, 74, 93, 331, 11, 16, 277, 138, 358, 246, 60, 322, 273, 75, 161, 235, 373, 193, 347, 341, 173, 140, 261, 0, 47, 13, 167, 292, 123, 258, 181, 361, 87, 236, 166, 252, 168, 34, 266, 96, 297, 176, 69, 226, 143, 19, 112, 182, 99, 56, 363, 174, 154, 298, 104, 204, 342, 142, 242, 317, 364, 58, 41, 378, 76, 218, 202, 333, 118, 281, 200, 201, 368, 83, 326, 36, 4, 21, 175, 164, 229, 288, 302, 172, 32, 119, 321, 380, 294, 85, 318, 53, 128, 339, 20, 269, 28, 127, 67, 130, 186, 316, 208, 374, 107, 313, 306, 325, 49, 286, 9, 77, 233, 111, 157, 126, 217, 79, 155, 106, 169, 291, 304, 98, 86, 100, 232, 89, 68, 51, 135, 64, 210, 323, 91, 256, 101, 303, 285, 346, 262, 211, 349, 350, 160, 220, 149, 121, 207, 203, 170, 268, 71, 48, 141, 295, 375, 265, 370, 105] + 9: [92, 186, 30, 259, 8, 339, 56, 263, 288, 183, 131, 75, 350, 228, 222, 184, 120, 98, 174, 180, 60, 167, 74, 311, 364, 119, 110, 323, 46, 65, 9, 99, 250, 343, 306, 315, 153, 81, 328, 262, 62, 318, 251, 195, 181, 378, 22, 163, 216, 330, 166, 38, 23, 31, 86, 194, 127, 309, 61, 133, 358, 232, 190, 287, 264, 185, 370, 290, 152, 44, 267, 349, 108, 204, 269, 238, 223, 95, 93, 173, 340, 295, 374, 169, 225, 373, 7, 66, 118, 43, 212, 336, 375, 149, 326, 240, 327, 353, 281, 132, 322, 35, 210, 192, 187, 170, 94, 129, 54, 324, 159, 29, 342, 314, 176, 172, 71, 144, 217, 58, 126, 70, 256, 199, 32, 218, 130, 16, 198, 76, 354, 73, 206, 227, 310, 123, 301, 366, 302, 164, 211, 27, 377, 252, 88, 379, 272, 21, 78, 2, 316, 233, 313, 33, 113, 91, 325, 168, 182, 96, 368, 137, 258, 329, 297, 361, 160, 77, 268, 352, 230, 89, 345, 294, 162, 300, 347, 67, 249, 236, 254, 175, 338, 265, 50, 4, 308, 201, 291, 202, 284, 26, 3, 331, 143, 355, 237, 17, 103, 191, 360, 36, 299, 219, 72, 20, 25, 148, 83, 231, 376, 253, 277, 307, 112, 372, 135, 296, 380, 381, 245, 111, 261, 128, 114, 208, 107, 156, 101, 97, 84, 134, 165, 117, 177, 178, 121, 155, 1, 18, 48, 64, 68, 157, 197, 12, 356, 273, 79, 285, 200, 154, 214, 109, 292, 39, 317, 41, 362, 209, 257, 146, 303, 193, 45, 52, 85, 205, 49, 226, 40, 141, 19, 63, 346, 53, 196, 348, 276, 282, 246, 279, 24, 122, 312, 55, 283, 57, 215, 260, 150, 138, 102, 80, 5, 104, 298, 239, 10, 321, 224, 87, 244, 371, 359, 221, 47, 289, 335, 274, 220, 37, 13, 139, 6, 275, 341, 367, 151, 363, 333, 344, 235, 243, 42, 125, 242, 247, 351, 382, 179, 337, 105, 14, 320, 189, 229, 207, 293, 136, 124, 203, 332, 213, 82, 255, 357, 0, 106, 278, 171, 234, 145, 100, 15, 369, 142, 280, 304, 266, 286, 69, 383, 140, 241, 34, 271, 334, 319, 188, 365, 270, 248, 305, 28, 90, 115, 158, 59, 147, 51, 116, 11, 161] + 10: [55, 61, 222, 353, 78, 262, 365, 69, 234, 227, 124, 293, 246, 289, 138, 8, 282, 109, 341, 113, 231, 181, 338, 40, 132, 223, 257, 306, 72, 116, 75, 184, 130, 240, 267, 148, 123, 220, 327, 188, 114, 210, 243, 256, 137, 214, 189, 252, 330, 121, 100, 37, 190, 265, 308, 297, 145, 153, 131, 346, 88, 167, 122, 202, 328, 313, 383, 26, 335, 93, 13, 52, 118, 381, 323, 261, 296, 260, 134, 147, 270, 303, 366, 173, 162, 336, 166, 233, 216, 369, 64, 357, 291, 209, 334, 292, 31, 321, 180, 91, 378, 73, 141, 249, 229, 281, 102, 59, 107, 199, 301, 105, 251, 170, 149, 94, 280, 354, 352, 279, 164, 111, 87, 304, 195, 219, 343, 32, 0, 363, 376, 161, 225, 203, 259, 95, 242, 367, 67, 187, 9, 171, 119, 117, 80, 263, 53, 16, 10, 333, 43, 54, 18, 41, 269, 298, 226, 294, 344, 355, 320, 213, 276, 35, 205, 255, 51, 272, 142, 197, 360, 200, 70, 299, 12, 198, 285, 120, 28, 311, 165, 89, 288, 155, 63, 151, 6, 68, 314, 183, 332, 339, 5, 250, 66, 309, 2, 258, 33, 144, 92, 264, 318, 90, 191, 215, 268, 337, 310, 248, 239, 74, 110, 271, 284, 221, 204, 156, 382, 317, 169, 283, 377, 127, 351, 302, 361, 182, 375, 211, 17, 373, 217, 76, 20, 96, 7, 15, 56, 207, 129, 380, 286, 84, 60, 368, 370, 364, 201, 325, 235, 237, 275, 290, 30, 44, 372, 295, 374, 196, 98, 125, 307, 287, 228, 238, 146, 186, 85, 324, 1, 79, 348, 58, 4, 193, 305, 38, 82, 340, 83, 86, 254, 23, 29, 350, 157, 277, 319, 115, 266, 192, 224, 177, 103, 345, 218, 247, 99, 158, 97, 57, 342, 108, 11, 274, 241, 101, 81, 65, 22, 106, 230, 326, 322, 152, 19, 174, 128, 316, 133, 315, 329, 126, 245, 143, 185, 379, 356, 3, 34, 25, 212, 312, 154, 47, 140, 362, 49, 112, 163, 24, 359, 62, 50, 77, 150, 45, 194, 206, 208, 175, 48, 300, 172, 349, 371, 139, 159, 21, 278, 104, 27, 39, 36, 253, 358, 71, 46, 168, 14, 347, 135, 331, 236, 178, 160, 244, 179, 176, 136, 232, 273, 42] + 11: [377, 4, 277, 186, 249, 60, 317, 107, 230, 143, 150, 374, 278, 95, 131, 217, 321, 298, 293, 69, 1, 280, 165, 59, 52, 187, 67, 285, 360, 160, 294, 250, 85, 282, 297, 267, 64, 37, 253, 256, 315, 25, 198, 255, 7, 279, 40, 91, 142, 370, 332, 367, 133, 199, 222, 261, 62, 364, 292, 363, 11, 63, 144, 151, 49, 32, 311, 174, 324, 197, 113, 304, 21, 192, 158, 41, 88, 163, 193, 289, 136, 206, 361, 175, 139, 48, 270, 226, 152, 8, 352, 371, 241, 355, 55, 348, 130, 203, 17, 339, 196, 128, 313, 271, 325, 66, 247, 349, 166, 101, 87, 378, 242, 123, 331, 219, 171, 356, 53, 168, 189, 28, 182, 376, 71, 216, 232, 248, 75, 92, 207, 35, 149, 93, 350, 375, 170, 24, 372, 214, 46, 81, 237, 20, 58, 223, 36, 51, 98, 329, 47, 141, 184, 57, 245, 266, 167, 19, 319, 109, 129, 12, 301, 134, 296, 177, 18, 290, 169, 183, 103, 263, 354, 191, 236, 288, 268, 155, 138, 260, 210, 383, 100, 148, 202, 265, 105, 72, 308, 33, 112, 120, 156, 234, 83, 110, 362, 326, 254, 213, 208, 194, 30, 238, 300, 220, 299, 264, 99, 10, 188, 27, 302, 291, 281, 172, 82, 45, 357, 345, 74, 330, 31, 34, 259, 190, 2, 157, 201, 369, 318, 185, 347, 159, 42, 84, 287, 275, 209, 153, 272, 68, 273, 231, 342, 161, 200, 106, 6, 212, 137, 127, 337, 116, 79, 276, 70, 50, 86, 303, 314, 77, 224, 76, 221, 38, 108, 366, 5, 295, 335, 351, 78, 135, 320, 286, 327, 205, 328, 102, 22, 215, 344, 96, 338, 181, 379, 9, 284, 29, 306, 145, 235, 56, 43, 258, 233, 121, 80, 228, 316, 26, 373, 309, 23, 322, 90, 119, 44, 180, 115, 65, 97, 173, 162, 211, 195, 229, 39, 154, 243, 251, 239, 218, 146, 164, 73, 380, 368, 176, 365, 257, 343, 125, 341, 381, 353, 111, 274, 336, 305, 340, 0, 246, 178, 104, 227, 3, 240, 117, 16, 54, 204, 310, 252, 94, 118, 307, 346, 358, 179, 89, 124, 147, 122, 283, 382, 312, 244, 140, 269, 15, 13, 262, 333, 359, 323, 225, 126, 132, 334, 114, 61, 14] + 12: [226, 20, 194, 300, 120, 298, 109, 71, 3, 304, 108, 185, 280, 39, 59, 127, 237, 9, 198, 227, 268, 294, 93, 86, 330, 41, 329, 245, 175, 49, 317, 148, 161, 96, 196, 151, 331, 69, 236, 52, 335, 53, 267, 88, 332, 92, 114, 381, 232, 338, 25, 33, 366, 363, 77, 371, 374, 130, 137, 85, 170, 31, 191, 351, 1, 79, 178, 10, 19, 216, 307, 24, 5, 115, 243, 12, 143, 375, 188, 252, 122, 149, 167, 179, 80, 16, 23, 354, 314, 42, 213, 259, 211, 352, 220, 285, 82, 292, 308, 218, 203, 90, 234, 51, 140, 337, 72, 172, 281, 373, 207, 370, 327, 320, 113, 233, 75, 286, 219, 125, 293, 166, 107, 155, 246, 176, 180, 301, 76, 276, 258, 102, 55, 289, 156, 362, 43, 94, 275, 359, 200, 2, 173, 18, 224, 206, 136, 240, 265, 202, 123, 63, 299, 353, 132, 279, 356, 382, 89, 100, 128, 57, 101, 134, 54, 310, 262, 215, 30, 334, 135, 323, 325, 221, 35, 269, 347, 44, 119, 67, 228, 74, 56, 27, 368, 217, 249, 117, 66, 165, 60, 162, 204, 341, 255, 112, 306, 378, 13, 247, 138, 346, 254, 208, 305, 355, 195, 110, 186, 361, 326, 22, 312, 64, 62, 129, 251, 193, 11, 87, 210, 263, 38, 40, 153, 222, 192, 144, 288, 169, 328, 164, 28, 177, 26, 48, 309, 111, 376, 116, 273, 163, 6, 383, 34, 365, 257, 379, 291, 32, 350, 160, 189, 121, 340, 318, 238, 159, 360, 84, 239, 266, 260, 98, 242, 61, 65, 142, 364, 278, 73, 146, 197, 70, 372, 377, 183, 58, 324, 336, 253, 342, 15, 154, 212, 303, 272, 145, 78, 287, 150, 158, 68, 50, 322, 147, 315, 201, 209, 133, 313, 81, 271, 270, 367, 230, 290, 274, 168, 36, 46, 103, 4, 344, 380, 321, 106, 99, 105, 21, 283, 229, 91, 174, 8, 250, 282, 277, 345, 45, 214, 181, 37, 357, 264, 316, 296, 311, 29, 83, 244, 14, 284, 205, 248, 339, 297, 190, 187, 235, 369, 124, 95, 302, 0, 171, 118, 97, 152, 256, 141, 343, 223, 131, 319, 7, 241, 126, 182, 47, 17, 348, 231, 139, 184, 225, 295, 333, 104, 358, 199, 261, 349, 157] + 13: [108, 46, 172, 186, 329, 340, 231, 363, 199, 316, 134, 228, 77, 16, 55, 244, 110, 86, 324, 318, 36, 6, 125, 201, 83, 348, 109, 277, 282, 314, 40, 335, 37, 266, 121, 364, 275, 333, 21, 205, 261, 61, 322, 300, 249, 81, 238, 260, 66, 339, 326, 288, 253, 50, 92, 285, 243, 270, 79, 38, 317, 360, 63, 279, 47, 240, 67, 303, 33, 1, 26, 115, 220, 194, 251, 357, 268, 283, 334, 310, 371, 367, 245, 53, 48, 168, 17, 44, 215, 258, 27, 129, 350, 341, 132, 302, 170, 328, 309, 179, 87, 273, 30, 373, 379, 319, 72, 0, 382, 200, 291, 307, 76, 184, 188, 353, 139, 153, 11, 234, 52, 198, 177, 35, 207, 267, 222, 185, 84, 25, 169, 352, 223, 226, 41, 361, 2, 297, 107, 94, 45, 305, 126, 144, 203, 128, 162, 164, 269, 88, 290, 362, 187, 299, 101, 232, 263, 225, 202, 74, 247, 190, 182, 271, 289, 209, 365, 359, 212, 10, 193, 127, 295, 150, 7, 280, 327, 3, 112, 28, 122, 167, 98, 105, 192, 29, 354, 381, 343, 377, 32, 59, 23, 298, 159, 8, 311, 111, 208, 151, 65, 211, 214, 284, 191, 369, 123, 338, 345, 91, 315, 31, 147, 149, 221, 248, 69, 135, 131, 141, 64, 210, 293, 272, 346, 296, 292, 213, 294, 197, 229, 89, 380, 176, 286, 224, 82, 24, 281, 257, 161, 54, 15, 183, 39, 233, 265, 100, 4, 57, 133, 137, 254, 113, 173, 347, 216, 22, 287, 375, 104, 73, 136, 374, 320, 13, 372, 330, 206, 140, 306, 241, 20, 142, 276, 78, 90, 155, 204, 181, 332, 250, 124, 130, 175, 378, 85, 152, 189, 12, 195, 102, 117, 304, 56, 358, 58, 342, 178, 93, 259, 116, 51, 143, 196, 239, 171, 321, 166, 96, 34, 97, 19, 337, 75, 355, 370, 323, 313, 246, 227, 174, 237, 68, 218, 71, 376, 301, 62, 242, 99, 42, 120, 154, 49, 180, 14, 236, 114, 119, 344, 219, 80, 43, 308, 5, 274, 264, 9, 138, 163, 230, 160, 235, 252, 312, 146, 157, 349, 368, 106, 118, 156, 336, 60, 256, 148, 325, 356, 262, 158, 18, 103, 351, 366, 255, 95, 165, 278, 383, 145, 331, 70, 217] + 14: [4, 147, 30, 14, 95, 291, 118, 34, 17, 90, 356, 215, 158, 23, 92, 62, 343, 342, 145, 284, 180, 324, 369, 312, 64, 16, 317, 250, 274, 272, 115, 134, 359, 51, 262, 252, 36, 325, 224, 368, 183, 160, 285, 378, 290, 87, 24, 382, 381, 89, 277, 102, 94, 232, 57, 348, 333, 303, 136, 276, 365, 351, 119, 1, 82, 367, 98, 32, 165, 88, 74, 194, 326, 223, 96, 231, 211, 3, 323, 18, 28, 172, 146, 293, 263, 171, 138, 42, 46, 176, 195, 20, 111, 72, 106, 120, 204, 164, 110, 261, 151, 184, 213, 127, 238, 91, 375, 142, 141, 313, 139, 355, 362, 75, 71, 206, 58, 298, 220, 301, 268, 318, 38, 334, 199, 25, 275, 222, 257, 376, 198, 269, 345, 86, 60, 170, 167, 254, 350, 270, 280, 190, 66, 309, 247, 22, 129, 202, 152, 366, 173, 322, 300, 209, 246, 240, 187, 191, 329, 236, 225, 321, 93, 7, 192, 255, 135, 282, 193, 357, 133, 201, 159, 319, 320, 346, 84, 245, 85, 143, 39, 360, 336, 260, 363, 178, 289, 100, 188, 216, 140, 181, 70, 126, 163, 383, 2, 128, 8, 40, 349, 227, 5, 79, 80, 354, 44, 63, 149, 125, 234, 150, 217, 273, 109, 308, 56, 379, 296, 104, 116, 6, 361, 239, 278, 101, 226, 352, 200, 197, 248, 305, 169, 76, 105, 97, 212, 218, 182, 373, 307, 11, 54, 267, 327, 253, 137, 364, 335, 338, 286, 251, 130, 81, 205, 243, 21, 340, 153, 259, 26, 235, 256, 47, 59, 131, 249, 315, 265, 132, 48, 108, 10, 310, 229, 53, 112, 228, 304, 68, 113, 203, 50, 65, 210, 73, 337, 295, 15, 117, 122, 344, 287, 13, 271, 154, 377, 177, 370, 294, 99, 244, 156, 69, 242, 281, 380, 237, 279, 374, 347, 330, 161, 83, 341, 372, 186, 175, 283, 49, 266, 208, 157, 52, 185, 103, 162, 41, 207, 311, 297, 358, 123, 332, 77, 230, 35, 107, 124, 214, 0, 299, 19, 241, 43, 233, 33, 258, 168, 37, 55, 196, 353, 9, 316, 306, 288, 61, 148, 114, 155, 166, 144, 27, 264, 292, 314, 121, 302, 12, 221, 29, 67, 219, 371, 179, 189, 31, 45, 78, 331, 174, 339, 328] + 15: [11, 353, 24, 168, 128, 56, 344, 329, 265, 13, 173, 231, 284, 10, 145, 267, 273, 96, 257, 238, 63, 213, 84, 161, 382, 74, 2, 71, 282, 7, 122, 305, 317, 224, 266, 285, 249, 380, 303, 343, 377, 372, 188, 254, 274, 186, 70, 126, 34, 46, 379, 152, 64, 356, 142, 191, 360, 192, 318, 327, 44, 9, 85, 331, 205, 3, 123, 270, 118, 129, 89, 75, 21, 347, 132, 260, 314, 110, 154, 146, 0, 247, 354, 310, 237, 229, 239, 6, 223, 345, 374, 103, 61, 29, 328, 208, 362, 232, 359, 38, 376, 253, 197, 8, 133, 319, 246, 212, 109, 290, 134, 150, 114, 51, 16, 262, 309, 80, 258, 88, 138, 291, 251, 31, 272, 158, 54, 297, 332, 67, 203, 361, 187, 233, 346, 378, 202, 321, 235, 90, 250, 165, 40, 174, 119, 330, 198, 55, 53, 95, 1, 299, 112, 236, 25, 338, 92, 289, 184, 211, 12, 77, 121, 37, 227, 315, 226, 170, 127, 264, 69, 141, 322, 276, 23, 375, 201, 143, 45, 325, 62, 30, 140, 189, 179, 182, 139, 159, 218, 336, 176, 286, 15, 199, 28, 190, 200, 117, 350, 308, 5, 48, 252, 14, 94, 35, 39, 351, 326, 167, 334, 296, 248, 106, 66, 381, 280, 111, 65, 293, 68, 97, 169, 300, 204, 256, 306, 108, 153, 183, 155, 196, 365, 86, 164, 216, 302, 120, 207, 244, 107, 339, 73, 368, 221, 156, 185, 163, 316, 342, 50, 281, 72, 383, 181, 98, 349, 225, 60, 271, 57, 49, 341, 220, 367, 295, 312, 195, 259, 355, 58, 288, 166, 131, 81, 78, 292, 275, 18, 137, 219, 100, 136, 358, 144, 268, 269, 42, 151, 105, 217, 115, 313, 255, 324, 352, 148, 83, 240, 76, 4, 335, 357, 298, 320, 215, 277, 307, 193, 261, 20, 52, 340, 113, 373, 36, 32, 172, 209, 99, 26, 116, 371, 241, 287, 263, 177, 333, 82, 33, 149, 43, 214, 243, 311, 157, 206, 323, 171, 175, 59, 102, 147, 364, 337, 234, 230, 101, 41, 242, 369, 135, 366, 104, 222, 22, 194, 370, 294, 87, 130, 160, 279, 27, 124, 178, 301, 304, 210, 228, 79, 93, 245, 125, 283, 278, 348, 180, 162, 91, 363, 17, 47, 19] + 16: [15, 374, 287, 348, 310, 134, 216, 301, 189, 92, 11, 379, 346, 331, 56, 135, 233, 305, 241, 9, 286, 187, 274, 185, 360, 212, 285, 342, 252, 238, 357, 315, 248, 129, 102, 57, 139, 227, 278, 265, 12, 246, 222, 340, 306, 220, 253, 194, 79, 131, 270, 191, 4, 314, 363, 288, 177, 329, 132, 181, 105, 334, 157, 138, 89, 197, 58, 170, 317, 365, 259, 35, 381, 325, 153, 341, 124, 63, 313, 37, 249, 322, 168, 299, 345, 359, 112, 48, 165, 179, 284, 376, 51, 309, 163, 3, 262, 100, 64, 183, 119, 260, 106, 71, 275, 330, 96, 370, 173, 180, 27, 280, 171, 182, 121, 298, 115, 93, 19, 213, 123, 358, 200, 0, 337, 31, 167, 29, 160, 300, 104, 80, 294, 2, 371, 10, 349, 47, 236, 70, 266, 207, 304, 87, 308, 268, 130, 225, 26, 323, 210, 316, 292, 28, 352, 144, 159, 269, 373, 297, 215, 99, 148, 40, 326, 78, 372, 81, 41, 235, 18, 136, 364, 272, 324, 23, 145, 195, 244, 126, 1, 263, 251, 223, 6, 117, 34, 30, 65, 303, 50, 156, 312, 5, 46, 118, 250, 59, 333, 14, 84, 205, 229, 44, 109, 281, 162, 375, 32, 75, 273, 217, 267, 283, 120, 66, 73, 52, 147, 211, 175, 336, 154, 54, 68, 158, 256, 214, 378, 88, 176, 151, 60, 143, 17, 234, 61, 141, 245, 186, 240, 36, 133, 83, 203, 276, 380, 13, 218, 230, 271, 282, 343, 289, 295, 258, 226, 178, 101, 62, 296, 383, 247, 356, 361, 74, 219, 169, 122, 368, 174, 38, 161, 140, 208, 264, 353, 366, 193, 33, 351, 261, 350, 192, 328, 367, 257, 291, 152, 114, 7, 198, 237, 277, 125, 369, 362, 85, 21, 43, 293, 188, 146, 347, 232, 255, 91, 172, 76, 354, 90, 142, 311, 97, 302, 24, 321, 202, 224, 42, 22, 199, 86, 243, 332, 16, 290, 8, 164, 382, 204, 209, 39, 166, 113, 128, 110, 190, 103, 25, 95, 108, 116, 307, 149, 20, 206, 111, 254, 82, 221, 335, 155, 67, 319, 320, 53, 242, 327, 355, 94, 184, 49, 338, 339, 98, 55, 231, 107, 150, 201, 69, 72, 196, 377, 239, 77, 228, 127, 45, 279, 137, 318, 344] + 17: [119, 188, 46, 374, 142, 275, 79, 146, 104, 243, 207, 56, 91, 24, 272, 356, 98, 97, 189, 284, 156, 116, 363, 94, 81, 310, 319, 20, 15, 71, 259, 171, 48, 222, 33, 328, 145, 58, 230, 186, 276, 209, 247, 158, 40, 360, 359, 11, 292, 301, 256, 105, 213, 53, 216, 200, 114, 187, 313, 172, 265, 218, 199, 255, 69, 18, 221, 239, 67, 164, 149, 327, 381, 252, 353, 132, 268, 17, 85, 30, 182, 306, 295, 264, 122, 165, 152, 224, 111, 235, 280, 312, 203, 166, 223, 304, 220, 236, 89, 136, 294, 44, 117, 361, 143, 138, 178, 118, 366, 194, 107, 167, 128, 22, 113, 19, 240, 278, 208, 131, 376, 12, 348, 23, 279, 135, 286, 38, 125, 238, 73, 206, 297, 248, 174, 289, 370, 317, 316, 78, 341, 63, 83, 127, 133, 244, 1, 334, 9, 90, 274, 55, 109, 288, 139, 228, 245, 144, 300, 233, 21, 241, 151, 309, 323, 150, 141, 162, 346, 283, 34, 74, 153, 217, 333, 242, 345, 198, 175, 159, 7, 371, 329, 102, 263, 43, 342, 50, 28, 57, 227, 358, 261, 337, 357, 100, 307, 80, 197, 76, 254, 193, 296, 380, 173, 378, 190, 336, 308, 115, 163, 258, 303, 3, 96, 140, 8, 367, 364, 231, 0, 5, 59, 101, 52, 362, 318, 185, 204, 232, 282, 234, 124, 298, 343, 375, 61, 36, 37, 45, 176, 382, 285, 352, 344, 126, 372, 16, 302, 66, 10, 351, 87, 39, 92, 331, 321, 330, 281, 14, 51, 84, 325, 311, 237, 88, 270, 72, 195, 160, 70, 147, 184, 161, 202, 349, 183, 291, 299, 215, 180, 68, 210, 169, 251, 120, 368, 322, 266, 212, 108, 335, 324, 177, 110, 157, 75, 214, 262, 99, 219, 2, 42, 314, 27, 305, 196, 315, 65, 129, 290, 26, 179, 347, 77, 62, 103, 250, 82, 340, 60, 93, 6, 168, 130, 260, 326, 293, 355, 225, 49, 271, 148, 191, 287, 154, 377, 134, 112, 226, 47, 253, 365, 273, 246, 267, 369, 155, 13, 338, 4, 95, 121, 137, 229, 64, 86, 269, 350, 332, 181, 29, 211, 32, 54, 249, 170, 339, 25, 373, 277, 257, 383, 35, 320, 192, 41, 123, 379, 354, 205, 201, 31, 106] + 18: [271, 43, 46, 30, 29, 375, 380, 346, 347, 142, 174, 328, 339, 171, 234, 338, 334, 203, 355, 195, 155, 210, 231, 168, 212, 366, 316, 257, 245, 213, 291, 242, 320, 131, 341, 10, 359, 280, 238, 67, 343, 286, 99, 158, 337, 19, 26, 23, 248, 326, 173, 183, 175, 270, 156, 353, 159, 76, 299, 70, 285, 145, 133, 246, 0, 367, 1, 374, 143, 304, 2, 83, 91, 186, 219, 134, 268, 97, 151, 227, 352, 224, 92, 44, 95, 36, 150, 20, 45, 49, 59, 74, 75, 94, 101, 127, 260, 192, 35, 117, 118, 207, 228, 119, 266, 11, 342, 205, 217, 208, 356, 294, 122, 237, 373, 163, 124, 193, 96, 82, 324, 53, 236, 108, 33, 18, 22, 56, 166, 325, 12, 66, 78, 152, 382, 252, 349, 7, 62, 269, 28, 112, 39, 58, 225, 114, 170, 261, 37, 85, 305, 379, 106, 167, 214, 162, 48, 128, 289, 81, 313, 15, 9, 357, 301, 47, 182, 218, 13, 292, 120, 196, 276, 141, 265, 194, 298, 4, 364, 303, 232, 104, 350, 107, 222, 132, 161, 165, 187, 197, 220, 230, 139, 185, 169, 251, 34, 65, 68, 31, 284, 146, 189, 164, 16, 361, 77, 61, 149, 288, 300, 89, 275, 369, 297, 102, 282, 335, 41, 235, 264, 40, 93, 206, 370, 272, 333, 72, 123, 279, 60, 188, 360, 358, 345, 216, 42, 202, 14, 32, 79, 181, 250, 223, 51, 24, 121, 383, 54, 240, 157, 116, 372, 129, 377, 178, 177, 312, 281, 241, 64, 253, 233, 90, 259, 371, 221, 307, 52, 184, 21, 317, 179, 153, 130, 290, 332, 209, 3, 296, 254, 247, 258, 274, 293, 311, 314, 322, 138, 98, 71, 69, 113, 144, 376, 330, 331, 50, 255, 368, 211, 363, 351, 176, 365, 63, 229, 6, 103, 136, 80, 215, 110, 256, 148, 190, 84, 263, 306, 105, 329, 244, 262, 8, 199, 308, 38, 87, 86, 57, 267, 160, 319, 154, 201, 310, 243, 17, 239, 115, 111, 302, 283, 27, 135, 5, 278, 315, 200, 191, 147, 25, 172, 126, 277, 381, 109, 318, 100, 273, 348, 295, 140, 137, 204, 125, 198, 321, 88, 180, 226, 73, 378, 249, 55, 309, 287, 323, 327, 336, 340, 344, 354, 362] + 19: [24, 328, 207, 134, 323, 158, 152, 58, 141, 279, 339, 161, 354, 183, 243, 3, 165, 61, 208, 84, 302, 333, 147, 94, 342, 296, 360, 216, 217, 314, 352, 255, 44, 191, 239, 192, 202, 125, 230, 280, 331, 337, 41, 210, 182, 306, 370, 143, 317, 258, 371, 10, 212, 40, 330, 335, 5, 356, 324, 99, 81, 310, 76, 140, 151, 267, 11, 269, 341, 18, 234, 344, 123, 67, 254, 78, 168, 169, 128, 277, 170, 39, 276, 131, 23, 25, 32, 34, 46, 62, 72, 93, 98, 100, 106, 109, 190, 334, 363, 90, 188, 307, 115, 120, 1, 166, 287, 312, 251, 156, 297, 283, 96, 80, 377, 180, 240, 157, 359, 366, 260, 2, 73, 235, 320, 113, 13, 19, 92, 31, 144, 15, 219, 281, 378, 367, 358, 6, 55, 353, 146, 286, 221, 51, 107, 95, 111, 175, 112, 351, 110, 247, 130, 233, 293, 63, 364, 382, 127, 199, 285, 86, 28, 82, 209, 88, 308, 305, 381, 223, 278, 186, 298, 164, 350, 119, 313, 379, 129, 340, 236, 174, 347, 172, 118, 126, 133, 136, 137, 155, 163, 201, 187, 222, 97, 52, 244, 206, 77, 288, 91, 265, 203, 193, 132, 29, 215, 338, 315, 301, 8, 275, 300, 101, 38, 117, 374, 227, 167, 153, 321, 79, 272, 43, 176, 349, 159, 30, 249, 361, 290, 177, 53, 74, 295, 89, 195, 102, 200, 259, 14, 59, 65, 336, 16, 304, 228, 181, 224, 54, 27, 262, 60, 36, 26, 237, 69, 64, 45, 37, 85, 145, 252, 196, 0, 232, 21, 375, 104, 248, 362, 17, 213, 253, 343, 380, 12, 49, 250, 50, 75, 198, 205, 229, 245, 266, 274, 316, 256, 66, 7, 179, 154, 291, 211, 121, 33, 114, 122, 108, 9, 373, 318, 171, 71, 264, 270, 135, 47, 20, 284, 218, 116, 68, 138, 238, 139, 273, 303, 57, 83, 56, 184, 294, 173, 325, 311, 178, 242, 271, 149, 292, 231, 383, 241, 148, 160, 185, 70, 197, 22, 372, 194, 35, 204, 299, 282, 332, 345, 257, 368, 246, 42, 289, 189, 87, 124, 376, 142, 348, 162, 220, 329, 225, 346, 322, 261, 369, 268, 48, 226, 309, 150, 103, 4, 263, 105, 214, 319, 326, 327, 355, 357, 365] + 20: [37, 144, 369, 66, 223, 35, 252, 113, 291, 307, 339, 194, 177, 357, 374, 125, 216, 221, 289, 6, 256, 320, 188, 233, 33, 354, 90, 3, 323, 136, 346, 26, 264, 270, 110, 309, 227, 88, 50, 337, 164, 81, 312, 59, 31, 75, 133, 218, 46, 56, 141, 306, 253, 204, 48, 226, 298, 70, 308, 115, 340, 126, 217, 348, 73, 163, 238, 274, 5, 12, 16, 17, 23, 24, 28, 34, 38, 44, 47, 51, 52, 60, 62, 63, 67, 68, 72, 76, 84, 85, 89, 102, 103, 104, 107, 108, 278, 121, 311, 376, 248, 138, 318, 159, 41, 321, 145, 335, 236, 317, 4, 372, 245, 199, 64, 370, 124, 362, 375, 377, 371, 101, 213, 279, 228, 30, 45, 273, 119, 220, 240, 78, 171, 146, 334, 336, 180, 36, 282, 363, 210, 201, 219, 186, 355, 352, 65, 244, 80, 359, 0, 179, 15, 152, 61, 205, 69, 168, 301, 1, 79, 189, 239, 328, 109, 117, 118, 127, 129, 131, 134, 135, 139, 142, 147, 148, 149, 150, 151, 155, 156, 157, 167, 169, 170, 175, 181, 183, 190, 191, 192, 196, 116, 111, 122, 160, 324, 195, 325, 182, 187, 229, 351, 174, 58, 341, 292, 232, 99, 123, 158, 77, 259, 178, 209, 293, 284, 207, 246, 162, 93, 8, 140, 345, 14, 235, 184, 91, 13, 338, 267, 39, 130, 92, 258, 40, 185, 120, 7, 254, 18, 42, 272, 202, 54, 350, 222, 208, 55, 211, 74, 281, 83, 197, 343, 27, 96, 212, 251, 361, 198, 206, 214, 215, 230, 231, 234, 237, 241, 242, 243, 247, 249, 255, 257, 261, 262, 263, 266, 269, 271, 276, 277, 280, 283, 285, 286, 288, 381, 105, 161, 10, 82, 106, 2, 112, 97, 71, 290, 287, 353, 43, 95, 315, 193, 57, 25, 132, 380, 153, 342, 322, 364, 172, 378, 11, 86, 347, 200, 295, 49, 87, 165, 100, 302, 98, 114, 379, 32, 250, 19, 22, 319, 313, 368, 203, 173, 53, 305, 265, 9, 21, 154, 275, 268, 94, 128, 310, 143, 29, 176, 20, 166, 225, 365, 137, 224, 260, 366, 294, 296, 297, 299, 300, 303, 304, 314, 316, 326, 327, 329, 330, 331, 332, 333, 344, 349, 356, 358, 360, 367, 373, 382, 383] + 21: [371, 361, 250, 45, 262, 355, 245, 237, 141, 94, 29, 155, 365, 187, 25, 359, 367, 308, 162, 271, 159, 337, 351, 128, 287, 326, 132, 194, 43, 133, 22, 14, 98, 26, 183, 15, 383, 148, 354, 23, 336, 139, 358, 286, 85, 210, 9, 280, 44, 40, 324, 188, 203, 382, 161, 290, 191, 299, 156, 64, 229, 207, 363, 259, 289, 345, 123, 234, 166, 332, 35, 163, 310, 131, 243, 369, 212, 236, 223, 108, 157, 3, 7, 10, 11, 17, 32, 36, 42, 56, 58, 59, 61, 70, 93, 104, 158, 256, 270, 232, 112, 379, 277, 297, 66, 304, 213, 149, 364, 349, 55, 205, 233, 225, 150, 247, 208, 178, 189, 267, 374, 257, 252, 242, 5, 171, 258, 318, 4, 196, 227, 34, 238, 74, 316, 261, 97, 38, 263, 86, 312, 105, 380, 170, 126, 52, 303, 186, 248, 47, 119, 174, 314, 147, 192, 222, 91, 53, 117, 77, 21, 333, 30, 48, 114, 323, 330, 373, 130, 296, 99, 101, 90, 344, 334, 285, 73, 62, 68, 228, 217, 107, 125, 136, 164, 167, 179, 181, 184, 198, 215, 216, 152, 121, 214, 20, 200, 124, 230, 87, 175, 279, 218, 65, 135, 88, 240, 92, 377, 362, 103, 100, 82, 8, 295, 115, 276, 60, 211, 24, 113, 63, 346, 328, 173, 195, 190, 360, 226, 321, 255, 291, 224, 375, 102, 320, 120, 300, 209, 160, 28, 185, 84, 153, 67, 96, 1, 370, 313, 54, 302, 146, 50, 244, 110, 273, 357, 356, 76, 78, 41, 331, 39, 239, 6, 180, 353, 116, 151, 137, 197, 288, 251, 246, 127, 134, 37, 122, 294, 219, 264, 265, 266, 269, 272, 278, 282, 292, 281, 51, 182, 254, 80, 2, 199, 284, 352, 172, 221, 220, 71, 378, 376, 144, 19, 129, 293, 89, 348, 143, 12, 46, 33, 13, 325, 169, 16, 341, 366, 275, 18, 368, 75, 274, 342, 301, 83, 206, 27, 204, 372, 0, 140, 118, 168, 165, 202, 176, 260, 106, 340, 347, 142, 249, 298, 72, 109, 111, 338, 307, 57, 235, 69, 95, 81, 79, 283, 381, 317, 241, 309, 145, 177, 253, 31, 193, 231, 306, 268, 343, 154, 201, 49, 138, 329, 305, 311, 315, 319, 322, 327, 335, 339, 350] + 22: [73, 118, 329, 164, 226, 376, 51, 197, 15, 126, 170, 132, 251, 180, 305, 21, 330, 122, 179, 314, 205, 371, 184, 56, 59, 326, 352, 189, 68, 264, 255, 268, 347, 362, 69, 78, 292, 46, 19, 99, 322, 171, 232, 152, 138, 8, 85, 319, 369, 28, 340, 327, 277, 219, 18, 90, 137, 97, 168, 158, 301, 378, 26, 203, 0, 10, 140, 295, 1, 7, 12, 14, 30, 34, 35, 39, 40, 41, 43, 45, 47, 48, 49, 52, 55, 77, 81, 84, 86, 91, 93, 95, 96, 100, 102, 103, 209, 328, 224, 148, 153, 240, 216, 178, 80, 61, 139, 339, 299, 79, 215, 318, 221, 201, 252, 306, 214, 361, 50, 238, 151, 281, 293, 289, 58, 334, 37, 200, 66, 76, 38, 191, 294, 156, 227, 309, 272, 94, 92, 33, 188, 70, 22, 173, 6, 89, 364, 3, 218, 320, 379, 242, 32, 98, 287, 75, 36, 88, 206, 303, 2, 67, 235, 62, 150, 300, 104, 105, 107, 114, 119, 120, 121, 124, 128, 129, 131, 135, 136, 142, 143, 145, 146, 154, 157, 159, 161, 162, 165, 166, 167, 176, 343, 141, 357, 125, 348, 63, 24, 304, 111, 345, 110, 333, 198, 274, 351, 311, 54, 278, 244, 183, 42, 367, 193, 307, 174, 134, 72, 82, 123, 195, 366, 20, 273, 279, 11, 324, 363, 163, 5, 60, 115, 285, 283, 108, 269, 130, 365, 172, 263, 321, 25, 17, 31, 116, 249, 296, 53, 302, 335, 286, 64, 101, 213, 332, 13, 83, 239, 65, 233, 358, 181, 182, 186, 187, 190, 192, 194, 199, 202, 208, 217, 222, 223, 230, 234, 236, 237, 241, 243, 245, 247, 248, 250, 253, 256, 257, 317, 207, 260, 144, 147, 169, 204, 337, 177, 288, 74, 113, 261, 355, 381, 377, 175, 275, 331, 349, 246, 336, 211, 155, 27, 338, 228, 341, 312, 225, 220, 276, 133, 29, 229, 258, 109, 316, 254, 160, 87, 259, 106, 382, 360, 374, 23, 149, 9, 212, 4, 44, 298, 210, 57, 185, 380, 284, 383, 196, 117, 112, 231, 356, 16, 127, 325, 71, 270, 373, 262, 265, 266, 267, 271, 280, 282, 290, 291, 297, 308, 310, 313, 315, 323, 342, 344, 346, 350, 353, 354, 359, 368, 370, 372, 375] + 23: [172, 232, 374, 82, 86, 49, 170, 203, 123, 4, 299, 254, 279, 370, 100, 230, 292, 261, 1, 36, 132, 371, 336, 185, 217, 201, 219, 270, 125, 210, 349, 308, 361, 21, 282, 164, 341, 335, 0, 84, 296, 277, 30, 216, 354, 24, 151, 77, 97, 221, 145, 225, 91, 351, 15, 87, 169, 8, 206, 350, 284, 44, 41, 199, 149, 33, 209, 139, 278, 207, 96, 177, 37, 179, 150, 259, 188, 141, 345, 121, 120, 250, 303, 236, 75, 242, 13, 14, 29, 31, 64, 88, 110, 113, 157, 168, 26, 327, 5, 146, 274, 63, 339, 133, 295, 156, 340, 193, 180, 222, 78, 54, 329, 342, 297, 117, 38, 196, 346, 98, 50, 355, 147, 249, 247, 42, 127, 119, 122, 262, 32, 67, 239, 115, 108, 138, 287, 256, 109, 165, 101, 19, 154, 17, 306, 47, 103, 106, 45, 362, 365, 360, 233, 328, 255, 289, 23, 85, 271, 215, 40, 300, 227, 375, 280, 118, 128, 28, 382, 46, 311, 364, 160, 27, 211, 356, 372, 135, 61, 3, 316, 171, 114, 83, 89, 363, 198, 202, 214, 235, 265, 276, 167, 80, 358, 220, 111, 337, 197, 94, 253, 70, 130, 229, 309, 237, 184, 204, 52, 264, 307, 218, 74, 379, 373, 48, 53, 290, 324, 315, 331, 107, 263, 79, 353, 10, 73, 240, 245, 93, 269, 321, 228, 182, 7, 51, 258, 344, 191, 190, 367, 99, 159, 326, 174, 241, 55, 140, 267, 377, 153, 334, 20, 238, 305, 369, 9, 347, 266, 12, 137, 275, 381, 183, 60, 333, 260, 43, 318, 298, 205, 310, 293, 243, 294, 56, 57, 223, 192, 383, 212, 272, 134, 268, 288, 291, 301, 319, 352, 131, 314, 187, 176, 148, 213, 173, 72, 378, 102, 359, 285, 158, 58, 2, 376, 92, 155, 35, 273, 143, 313, 162, 332, 11, 322, 208, 6, 152, 195, 22, 304, 81, 163, 338, 317, 312, 161, 244, 323, 90, 124, 325, 257, 105, 104, 348, 248, 76, 226, 380, 283, 178, 234, 25, 252, 200, 66, 166, 62, 95, 320, 302, 129, 186, 116, 112, 343, 175, 142, 68, 224, 194, 246, 126, 59, 71, 18, 16, 144, 231, 65, 281, 181, 34, 357, 69, 39, 251, 136, 189, 286, 330, 366, 368] + 24: [70, 213, 122, 220, 237, 1, 226, 211, 202, 316, 210, 371, 257, 12, 182, 178, 381, 93, 8, 6, 41, 38, 382, 50, 258, 114, 181, 37, 88, 204, 262, 110, 203, 18, 357, 188, 345, 31, 306, 61, 366, 168, 347, 358, 218, 349, 155, 215, 108, 326, 145, 275, 286, 183, 310, 130, 185, 373, 106, 243, 90, 40, 47, 184, 285, 153, 120, 105, 305, 162, 298, 100, 376, 55, 56, 271, 160, 225, 10, 23, 32, 34, 36, 45, 57, 69, 74, 75, 76, 83, 98, 113, 115, 125, 129, 132, 246, 343, 301, 131, 290, 375, 283, 244, 344, 325, 137, 97, 350, 139, 201, 200, 287, 330, 319, 144, 263, 63, 269, 147, 84, 308, 4, 297, 333, 27, 190, 221, 189, 248, 229, 295, 96, 20, 261, 156, 302, 255, 216, 87, 157, 360, 224, 163, 146, 327, 117, 377, 152, 317, 328, 140, 66, 242, 186, 284, 126, 208, 133, 372, 362, 119, 197, 165, 228, 127, 300, 16, 25, 214, 85, 179, 369, 177, 81, 167, 60, 359, 176, 227, 143, 149, 161, 166, 169, 170, 173, 194, 196, 217, 240, 241, 0, 46, 5, 109, 195, 266, 118, 121, 28, 82, 172, 35, 142, 104, 318, 135, 299, 231, 256, 150, 95, 9, 15, 264, 289, 116, 253, 128, 73, 324, 151, 159, 293, 26, 39, 273, 282, 14, 44, 79, 268, 277, 103, 361, 124, 192, 365, 58, 307, 171, 315, 72, 320, 348, 338, 91, 64, 280, 30, 233, 279, 89, 54, 3, 33, 335, 234, 340, 199, 123, 193, 21, 134, 223, 48, 232, 368, 29, 77, 219, 180, 68, 2, 99, 187, 239, 249, 250, 251, 267, 270, 278, 281, 291, 303, 312, 92, 274, 154, 311, 265, 259, 86, 337, 175, 354, 43, 370, 136, 7, 342, 212, 112, 230, 323, 341, 355, 374, 51, 336, 49, 352, 62, 276, 111, 78, 254, 80, 53, 252, 321, 294, 138, 52, 260, 42, 191, 65, 11, 22, 59, 332, 102, 334, 13, 207, 71, 288, 205, 380, 236, 209, 322, 206, 245, 164, 346, 238, 67, 148, 235, 247, 363, 339, 313, 329, 19, 101, 174, 304, 17, 272, 158, 292, 94, 296, 222, 24, 141, 107, 198, 309, 314, 331, 351, 353, 356, 364, 367, 378, 379, 383] + 25: [379, 328, 80, 307, 327, 351, 89, 131, 359, 37, 222, 188, 382, 238, 365, 51, 340, 299, 194, 204, 196, 77, 68, 82, 192, 224, 0, 244, 40, 272, 24, 242, 305, 54, 201, 363, 228, 260, 368, 78, 35, 259, 383, 308, 128, 282, 312, 170, 109, 302, 30, 300, 235, 288, 23, 1, 292, 345, 94, 186, 112, 61, 184, 86, 58, 135, 198, 321, 344, 182, 189, 171, 338, 208, 213, 169, 177, 255, 104, 207, 263, 356, 3, 43, 87, 91, 95, 99, 119, 123, 174, 219, 286, 326, 329, 358, 269, 334, 167, 231, 140, 149, 215, 101, 366, 250, 309, 217, 120, 29, 218, 310, 16, 252, 11, 180, 111, 316, 336, 343, 296, 33, 136, 39, 57, 15, 8, 113, 153, 117, 348, 195, 81, 59, 306, 70, 214, 65, 144, 133, 63, 337, 197, 53, 145, 246, 41, 60, 294, 175, 258, 283, 187, 240, 339, 173, 369, 129, 349, 251, 254, 75, 88, 34, 139, 199, 205, 239, 352, 121, 55, 372, 209, 126, 122, 22, 277, 364, 243, 206, 322, 342, 267, 21, 355, 67, 13, 178, 164, 12, 264, 362, 232, 315, 193, 168, 31, 317, 9, 319, 287, 253, 64, 314, 211, 375, 25, 2, 124, 32, 146, 98, 320, 143, 73, 229, 52, 298, 261, 381, 20, 200, 361, 230, 293, 79, 132, 265, 191, 156, 225, 247, 270, 237, 125, 172, 203, 62, 74, 324, 6, 202, 116, 102, 69, 221, 281, 274, 311, 159, 318, 49, 353, 367, 138, 148, 373, 165, 83, 137, 157, 341, 210, 18, 234, 158, 71, 227, 85, 284, 50, 14, 19, 163, 92, 26, 150, 325, 160, 155, 303, 127, 257, 220, 118, 374, 152, 301, 354, 107, 72, 56, 278, 271, 332, 162, 27, 5, 291, 161, 76, 114, 280, 48, 266, 256, 262, 333, 313, 377, 115, 371, 100, 304, 295, 285, 268, 347, 46, 190, 44, 183, 241, 42, 273, 17, 335, 38, 10, 47, 103, 7, 84, 212, 97, 66, 28, 279, 216, 110, 4, 297, 108, 330, 276, 179, 249, 185, 346, 130, 350, 360, 380, 376, 142, 357, 105, 290, 248, 96, 275, 154, 370, 331, 233, 236, 378, 93, 166, 36, 289, 181, 323, 45, 90, 106, 147, 176, 223, 151, 134, 245, 226, 141] + 26: [217, 249, 113, 92, 128, 48, 328, 195, 108, 237, 293, 93, 272, 371, 367, 67, 216, 277, 188, 224, 318, 135, 329, 30, 2, 221, 126, 74, 232, 148, 44, 11, 77, 266, 94, 76, 223, 127, 134, 378, 159, 152, 192, 20, 308, 9, 370, 103, 335, 117, 331, 353, 212, 171, 349, 375, 352, 376, 301, 66, 88, 101, 325, 351, 98, 157, 64, 57, 305, 10, 43, 187, 313, 1, 3, 4, 14, 19, 22, 23, 25, 28, 29, 35, 47, 52, 54, 56, 58, 60, 63, 68, 73, 81, 84, 85, 90, 281, 27, 142, 337, 262, 89, 254, 278, 300, 294, 215, 15, 204, 137, 310, 62, 359, 183, 256, 342, 279, 174, 269, 55, 364, 110, 239, 184, 34, 231, 124, 156, 186, 220, 315, 24, 69, 245, 107, 116, 176, 327, 280, 5, 317, 208, 163, 6, 291, 314, 296, 298, 198, 160, 13, 97, 302, 100, 206, 109, 123, 46, 211, 168, 12, 377, 191, 111, 36, 326, 86, 225, 324, 91, 96, 115, 119, 121, 122, 132, 139, 141, 143, 145, 147, 150, 153, 169, 180, 185, 193, 194, 197, 199, 203, 380, 304, 227, 271, 120, 173, 360, 125, 21, 290, 170, 102, 251, 140, 138, 83, 182, 78, 165, 179, 282, 265, 344, 80, 236, 196, 59, 202, 334, 356, 230, 238, 382, 32, 258, 343, 241, 222, 275, 41, 233, 114, 363, 65, 190, 8, 154, 229, 289, 7, 161, 40, 244, 16, 37, 316, 144, 50, 347, 205, 149, 164, 366, 248, 285, 71, 49, 379, 247, 75, 332, 105, 255, 345, 207, 209, 210, 213, 218, 219, 228, 234, 235, 240, 242, 243, 246, 250, 252, 257, 259, 261, 264, 267, 268, 270, 136, 181, 338, 276, 214, 118, 146, 200, 112, 172, 178, 53, 38, 31, 70, 106, 341, 151, 131, 39, 61, 95, 323, 339, 321, 226, 263, 167, 166, 306, 355, 357, 162, 177, 18, 311, 79, 33, 129, 45, 130, 383, 336, 158, 320, 51, 87, 286, 82, 292, 312, 361, 17, 374, 0, 307, 303, 155, 340, 201, 72, 26, 354, 104, 99, 369, 175, 365, 253, 260, 189, 42, 133, 284, 346, 273, 274, 283, 287, 288, 295, 297, 299, 309, 319, 322, 330, 333, 348, 350, 358, 362, 368, 372, 373, 381] + 27: [103, 325, 10, 228, 50, 305, 133, 304, 371, 140, 28, 151, 171, 269, 78, 29, 178, 111, 131, 374, 37, 27, 324, 296, 49, 323, 105, 244, 109, 58, 132, 121, 254, 95, 266, 288, 52, 114, 321, 306, 327, 64, 316, 261, 226, 62, 232, 134, 60, 76, 284, 335, 146, 202, 170, 301, 139, 1, 136, 345, 61, 164, 283, 219, 44, 147, 200, 355, 231, 172, 287, 166, 130, 91, 41, 275, 319, 17, 154, 315, 63, 148, 243, 169, 367, 8, 51, 53, 79, 96, 137, 182, 206, 212, 255, 257, 32, 360, 98, 347, 89, 113, 141, 42, 25, 168, 349, 127, 252, 197, 73, 370, 379, 224, 277, 356, 116, 150, 71, 190, 188, 125, 120, 351, 250, 292, 218, 6, 217, 242, 106, 238, 173, 308, 294, 193, 162, 70, 334, 75, 68, 369, 149, 365, 93, 233, 31, 357, 124, 350, 207, 180, 189, 330, 67, 38, 11, 87, 208, 271, 279, 340, 14, 184, 352, 235, 122, 145, 20, 297, 331, 358, 186, 227, 363, 326, 138, 346, 36, 156, 220, 336, 101, 161, 256, 286, 203, 39, 4, 163, 270, 302, 225, 214, 12, 23, 260, 251, 16, 329, 85, 337, 118, 21, 303, 128, 353, 104, 198, 55, 343, 88, 258, 354, 48, 159, 259, 267, 187, 90, 339, 348, 94, 83, 376, 13, 181, 320, 280, 115, 5, 237, 86, 341, 272, 3, 185, 82, 199, 264, 80, 248, 84, 229, 281, 215, 377, 56, 99, 33, 9, 282, 239, 45, 97, 195, 313, 192, 26, 191, 69, 15, 299, 194, 167, 276, 155, 373, 278, 312, 24, 157, 205, 201, 307, 152, 117, 74, 247, 160, 285, 92, 240, 129, 300, 236, 332, 338, 333, 81, 177, 66, 222, 175, 265, 234, 213, 72, 230, 54, 7, 381, 0, 273, 143, 18, 317, 328, 362, 249, 263, 47, 34, 245, 183, 110, 318, 383, 344, 65, 246, 30, 204, 368, 210, 102, 382, 380, 291, 298, 295, 290, 211, 112, 274, 142, 135, 179, 309, 262, 241, 144, 35, 77, 126, 359, 366, 123, 158, 174, 375, 293, 289, 176, 22, 196, 108, 119, 216, 107, 59, 165, 314, 253, 153, 223, 268, 322, 221, 2, 364, 19, 46, 57, 100, 40, 361, 310, 209, 311, 43, 342, 372, 378] + 28: [310, 312, 201, 285, 179, 67, 3, 21, 148, 22, 112, 324, 243, 133, 224, 51, 252, 210, 25, 237, 7, 227, 321, 327, 144, 366, 221, 375, 1, 320, 268, 218, 158, 378, 70, 368, 311, 162, 130, 187, 104, 297, 209, 207, 223, 214, 77, 247, 98, 337, 125, 260, 177, 193, 184, 316, 334, 269, 63, 15, 374, 296, 56, 65, 283, 295, 82, 190, 246, 9, 258, 188, 46, 18, 315, 10, 85, 372, 139, 354, 145, 254, 89, 73, 336, 34, 123, 27, 270, 54, 212, 279, 96, 147, 160, 257, 317, 121, 197, 97, 176, 66, 289, 356, 100, 182, 16, 24, 170, 52, 369, 267, 370, 90, 241, 303, 166, 57, 341, 116, 53, 110, 32, 350, 314, 228, 5, 64, 44, 259, 349, 213, 80, 274, 363, 128, 230, 383, 308, 59, 171, 167, 181, 191, 273, 211, 276, 286, 37, 322, 272, 149, 300, 88, 131, 81, 345, 265, 359, 234, 192, 156, 309, 361, 76, 159, 189, 6, 151, 60, 39, 284, 351, 244, 103, 373, 12, 132, 118, 292, 94, 36, 115, 275, 173, 29, 256, 301, 290, 294, 298, 313, 253, 26, 0, 250, 307, 196, 169, 71, 376, 13, 165, 48, 205, 72, 172, 183, 262, 288, 157, 380, 20, 357, 107, 134, 305, 318, 175, 174, 352, 264, 362, 95, 153, 219, 346, 333, 238, 239, 137, 117, 233, 367, 280, 43, 17, 91, 99, 114, 225, 331, 120, 266, 249, 277, 271, 122, 152, 371, 140, 178, 358, 216, 281, 79, 329, 323, 339, 69, 245, 23, 33, 31, 106, 232, 119, 236, 291, 40, 55, 78, 126, 62, 231, 47, 141, 93, 186, 150, 365, 11, 146, 338, 348, 342, 343, 355, 255, 220, 235, 135, 328, 199, 163, 41, 302, 30, 222, 195, 108, 168, 278, 203, 344, 204, 326, 155, 50, 109, 226, 194, 92, 248, 105, 200, 263, 42, 242, 217, 61, 154, 306, 287, 206, 161, 240, 185, 340, 379, 330, 113, 129, 143, 4, 102, 14, 49, 229, 293, 208, 347, 198, 19, 35, 28, 124, 136, 332, 68, 38, 180, 58, 111, 202, 282, 299, 325, 84, 364, 382, 319, 74, 215, 87, 45, 251, 101, 8, 335, 142, 138, 353, 127, 2, 304, 86, 83, 164, 75, 261, 360, 377, 381] + 29: [197, 83, 277, 346, 140, 11, 132, 218, 116, 301, 180, 73, 253, 190, 214, 86, 19, 185, 349, 22, 62, 286, 113, 300, 318, 139, 175, 145, 3, 101, 340, 81, 353, 276, 2, 275, 187, 203, 287, 350, 146, 303, 94, 115, 198, 229, 5, 126, 144, 160, 103, 166, 290, 307, 117, 56, 359, 37, 322, 257, 309, 281, 51, 155, 252, 77, 68, 12, 194, 249, 192, 335, 15, 130, 321, 246, 107, 31, 82, 376, 6, 8, 10, 13, 29, 47, 69, 71, 79, 91, 96, 121, 138, 164, 165, 178, 127, 357, 314, 283, 255, 232, 4, 269, 110, 245, 215, 207, 59, 244, 195, 339, 18, 172, 345, 120, 99, 206, 330, 50, 55, 90, 250, 34, 235, 278, 291, 271, 361, 295, 325, 317, 188, 125, 323, 57, 142, 65, 143, 137, 64, 292, 167, 254, 52, 263, 299, 30, 92, 184, 289, 39, 227, 179, 355, 199, 338, 61, 364, 363, 16, 26, 98, 348, 123, 157, 268, 226, 134, 328, 191, 216, 381, 141, 372, 284, 35, 0, 306, 326, 17, 28, 239, 334, 135, 181, 95, 20, 66, 201, 212, 224, 74, 243, 311, 294, 85, 365, 280, 270, 312, 362, 375, 341, 148, 53, 259, 14, 324, 119, 360, 293, 228, 231, 327, 344, 36, 87, 333, 25, 354, 374, 196, 186, 208, 104, 320, 304, 370, 251, 124, 105, 273, 236, 129, 262, 156, 72, 7, 221, 319, 154, 274, 111, 264, 161, 308, 233, 222, 75, 67, 225, 209, 237, 298, 168, 131, 21, 63, 379, 23, 242, 234, 1, 373, 42, 122, 70, 329, 315, 93, 9, 106, 238, 282, 136, 24, 152, 337, 49, 205, 27, 153, 247, 305, 240, 260, 261, 174, 351, 266, 97, 128, 88, 40, 58, 248, 297, 352, 118, 382, 162, 149, 33, 44, 367, 48, 366, 78, 43, 151, 210, 223, 60, 369, 183, 310, 267, 219, 313, 279, 258, 80, 46, 220, 347, 114, 177, 368, 163, 343, 54, 316, 200, 32, 133, 112, 272, 159, 332, 380, 296, 171, 76, 45, 256, 100, 182, 204, 173, 383, 213, 241, 230, 41, 147, 84, 265, 193, 211, 109, 38, 189, 302, 358, 170, 108, 158, 217, 102, 336, 342, 89, 150, 202, 356, 331, 371, 176, 169, 378, 285, 288, 377] + 30: [239, 62, 144, 166, 28, 213, 362, 230, 298, 337, 60, 24, 272, 253, 198, 48, 246, 358, 292, 378, 303, 116, 163, 260, 193, 184, 289, 12, 273, 361, 196, 153, 212, 27, 147, 327, 96, 86, 352, 235, 300, 162, 141, 115, 247, 256, 41, 122, 309, 324, 133, 328, 43, 156, 5, 112, 132, 91, 173, 329, 72, 107, 299, 209, 342, 179, 314, 307, 367, 8, 148, 252, 19, 368, 231, 93, 42, 366, 75, 16, 295, 178, 114, 192, 313, 98, 297, 30, 21, 31, 32, 34, 36, 69, 79, 81, 37, 296, 233, 111, 6, 195, 248, 265, 318, 109, 245, 249, 250, 223, 52, 187, 97, 68, 221, 288, 63, 293, 200, 308, 160, 151, 139, 92, 172, 263, 217, 359, 261, 7, 283, 341, 51, 110, 129, 183, 325, 312, 332, 274, 199, 291, 128, 113, 170, 150, 335, 174, 104, 13, 87, 316, 157, 370, 26, 67, 25, 319, 154, 108, 39, 267, 186, 365, 229, 372, 301, 131, 219, 18, 176, 340, 57, 136, 134, 380, 374, 9, 56, 117, 232, 47, 149, 4, 169, 82, 88, 100, 106, 119, 127, 145, 140, 137, 65, 50, 53, 262, 121, 78, 11, 258, 38, 90, 152, 77, 14, 281, 89, 171, 224, 346, 158, 242, 351, 349, 66, 211, 142, 29, 210, 1, 167, 125, 376, 241, 205, 138, 99, 234, 320, 348, 161, 3, 220, 20, 143, 373, 271, 302, 315, 306, 379, 286, 343, 282, 227, 257, 226, 40, 215, 331, 240, 55, 103, 338, 311, 95, 275, 76, 181, 350, 165, 182, 243, 326, 251, 177, 330, 323, 126, 268, 83, 194, 276, 382, 218, 334, 146, 322, 228, 180, 185, 189, 190, 201, 237, 259, 208, 222, 277, 254, 381, 206, 2, 46, 317, 35, 164, 344, 202, 225, 357, 135, 383, 64, 10, 266, 23, 321, 203, 284, 197, 105, 244, 294, 71, 44, 285, 371, 280, 124, 155, 102, 214, 94, 369, 353, 333, 130, 84, 347, 33, 279, 175, 15, 287, 336, 360, 58, 120, 74, 45, 339, 304, 0, 290, 59, 236, 70, 270, 364, 255, 85, 188, 207, 80, 17, 159, 355, 216, 356, 264, 123, 49, 118, 375, 61, 238, 204, 73, 101, 168, 305, 54, 191, 22, 310, 269, 278, 345, 354, 363, 377] + 31: [288, 361, 123, 160, 82, 339, 176, 88, 240, 284, 146, 169, 60, 86, 70, 378, 61, 375, 45, 335, 277, 296, 336, 227, 290, 287, 177, 190, 2, 40, 195, 96, 78, 62, 275, 244, 34, 122, 267, 143, 357, 93, 219, 111, 105, 173, 8, 52, 249, 364, 368, 334, 319, 158, 145, 19, 344, 234, 209, 117, 136, 194, 382, 255, 273, 116, 247, 302, 36, 17, 212, 118, 216, 85, 80, 370, 217, 213, 300, 13, 35, 329, 120, 165, 73, 239, 127, 228, 24, 203, 95, 181, 347, 175, 207, 301, 248, 299, 253, 112, 153, 326, 124, 260, 383, 186, 252, 59, 317, 321, 183, 377, 269, 307, 359, 338, 87, 3, 178, 223, 131, 140, 272, 263, 57, 47, 79, 238, 292, 220, 200, 188, 44, 243, 320, 128, 332, 109, 167, 27, 29, 346, 144, 65, 107, 257, 171, 66, 137, 106, 254, 304, 15, 291, 193, 81, 282, 328, 362, 180, 351, 310, 148, 198, 179, 265, 280, 129, 215, 1, 199, 197, 259, 226, 168, 185, 113, 142, 68, 51, 298, 149, 246, 10, 318, 210, 352, 11, 343, 54, 162, 67, 159, 365, 360, 309, 125, 108, 39, 222, 75, 314, 354, 241, 102, 134, 191, 206, 214, 201, 58, 98, 261, 72, 139, 49, 110, 281, 84, 278, 293, 232, 224, 182, 196, 38, 379, 147, 174, 274, 312, 6, 4, 16, 286, 130, 77, 353, 56, 271, 12, 33, 92, 43, 18, 285, 31, 325, 211, 20, 250, 155, 297, 306, 303, 163, 91, 311, 337, 152, 99, 184, 374, 104, 69, 340, 161, 100, 356, 205, 225, 251, 294, 218, 341, 187, 381, 322, 90, 114, 372, 235, 135, 262, 172, 126, 229, 89, 115, 121, 237, 380, 138, 156, 376, 221, 264, 231, 23, 245, 133, 55, 25, 202, 270, 305, 26, 32, 316, 313, 349, 366, 0, 330, 170, 236, 164, 9, 355, 373, 103, 22, 324, 189, 345, 48, 276, 208, 21, 83, 348, 371, 157, 192, 141, 119, 258, 233, 266, 327, 242, 94, 97, 71, 331, 154, 166, 150, 333, 53, 363, 256, 308, 30, 342, 295, 63, 5, 367, 74, 268, 7, 46, 64, 76, 323, 41, 279, 283, 289, 369, 50, 14, 151, 42, 132, 358, 315, 101, 350, 230, 204, 28, 37] + 32: [148, 89, 83, 86, 238, 33, 237, 166, 45, 218, 120, 307, 348, 179, 175, 105, 221, 16, 194, 7, 334, 282, 190, 293, 322, 151, 53, 77, 380, 88, 110, 254, 60, 212, 126, 319, 87, 76, 150, 41, 213, 340, 231, 240, 377, 79, 280, 69, 292, 1, 129, 306, 32, 369, 18, 195, 121, 270, 315, 331, 178, 233, 154, 9, 128, 114, 370, 52, 248, 226, 314, 133, 199, 374, 31, 354, 3, 176, 350, 259, 379, 366, 295, 162, 260, 267, 97, 74, 38, 242, 62, 235, 11, 78, 117, 147, 106, 108, 227, 347, 55, 4, 229, 336, 335, 48, 171, 202, 40, 249, 58, 258, 243, 200, 136, 113, 153, 273, 261, 241, 368, 209, 21, 96, 19, 371, 297, 323, 68, 360, 49, 268, 274, 82, 339, 247, 342, 345, 51, 364, 278, 294, 232, 107, 122, 329, 70, 101, 239, 356, 10, 201, 35, 100, 164, 337, 134, 34, 90, 266, 288, 223, 376, 355, 119, 262, 43, 224, 152, 98, 289, 308, 37, 183, 296, 93, 91, 118, 244, 302, 245, 2, 80, 299, 198, 251, 135, 316, 161, 169, 180, 264, 163, 184, 367, 63, 168, 281, 157, 359, 324, 276, 160, 284, 85, 191, 131, 250, 139, 142, 219, 332, 214, 187, 256, 141, 286, 193, 36, 230, 330, 149, 59, 317, 72, 211, 127, 326, 279, 186, 67, 23, 271, 203, 353, 351, 263, 8, 298, 349, 313, 12, 42, 215, 197, 103, 22, 145, 343, 73, 66, 210, 170, 39, 328, 167, 321, 310, 225, 320, 46, 125, 275, 64, 358, 216, 318, 309, 124, 13, 54, 333, 47, 28, 112, 253, 123, 300, 44, 192, 14, 255, 291, 338, 272, 277, 290, 301, 362, 344, 352, 26, 24, 172, 99, 382, 155, 174, 285, 325, 252, 92, 5, 146, 143, 228, 156, 111, 165, 283, 6, 361, 206, 144, 75, 0, 303, 130, 205, 182, 327, 365, 109, 189, 137, 30, 207, 346, 95, 208, 140, 61, 378, 115, 246, 159, 234, 29, 204, 217, 25, 287, 50, 222, 116, 104, 357, 158, 196, 132, 94, 65, 269, 138, 363, 311, 102, 341, 373, 188, 265, 71, 181, 17, 15, 173, 84, 236, 81, 57, 372, 185, 304, 257, 177, 305, 27, 56, 20, 220, 381, 312, 375, 383] + 33: [91, 169, 377, 344, 342, 307, 38, 158, 51, 87, 255, 101, 94, 288, 191, 62, 14, 134, 323, 305, 132, 25, 79, 188, 170, 45, 182, 149, 318, 358, 347, 22, 233, 52, 85, 109, 67, 361, 72, 163, 141, 258, 335, 340, 267, 295, 212, 380, 360, 264, 123, 2, 226, 20, 222, 187, 362, 337, 356, 23, 197, 18, 110, 351, 143, 155, 312, 11, 116, 370, 284, 330, 136, 228, 53, 63, 7, 208, 78, 219, 332, 353, 282, 243, 24, 365, 310, 227, 15, 44, 32, 71, 95, 105, 128, 185, 177, 234, 167, 154, 66, 328, 338, 303, 261, 139, 229, 135, 183, 98, 41, 339, 273, 150, 204, 213, 130, 37, 8, 90, 382, 334, 202, 160, 240, 291, 343, 174, 374, 363, 6, 179, 84, 369, 224, 294, 138, 111, 9, 168, 359, 118, 12, 92, 10, 271, 100, 74, 315, 215, 157, 203, 50, 259, 248, 193, 235, 88, 230, 147, 21, 176, 27, 327, 241, 30, 16, 120, 201, 283, 251, 218, 126, 194, 189, 252, 244, 345, 114, 198, 97, 47, 119, 69, 253, 56, 186, 223, 266, 287, 298, 306, 127, 190, 262, 280, 89, 232, 59, 4, 270, 286, 319, 237, 320, 260, 381, 348, 151, 125, 217, 61, 153, 104, 31, 354, 57, 172, 80, 297, 205, 199, 254, 60, 124, 355, 40, 148, 275, 42, 162, 268, 129, 166, 39, 77, 366, 281, 206, 17, 250, 70, 164, 352, 103, 256, 313, 152, 367, 178, 290, 137, 214, 296, 209, 102, 195, 207, 26, 28, 173, 145, 112, 285, 161, 33, 184, 324, 292, 272, 372, 265, 156, 49, 304, 211, 257, 274, 13, 142, 29, 302, 236, 245, 309, 317, 326, 333, 144, 64, 113, 376, 48, 115, 108, 210, 81, 316, 314, 322, 5, 35, 311, 73, 325, 192, 375, 36, 321, 54, 238, 308, 289, 133, 121, 379, 146, 93, 247, 200, 299, 3, 329, 58, 1, 196, 96, 159, 75, 331, 181, 131, 99, 231, 349, 364, 249, 180, 171, 276, 269, 19, 300, 46, 246, 107, 43, 263, 76, 122, 371, 86, 68, 83, 82, 279, 350, 117, 65, 165, 242, 55, 341, 378, 216, 368, 221, 346, 175, 239, 225, 106, 140, 278, 277, 220, 0, 383, 336, 301, 34, 293, 357, 373] + 34: [179, 239, 136, 360, 379, 74, 283, 81, 293, 331, 294, 188, 298, 192, 348, 260, 332, 65, 80, 85, 23, 79, 104, 190, 97, 110, 148, 195, 342, 326, 120, 232, 364, 245, 131, 324, 314, 221, 270, 382, 155, 146, 305, 27, 142, 355, 344, 184, 267, 92, 353, 114, 210, 278, 118, 199, 170, 319, 77, 187, 67, 241, 227, 176, 209, 13, 175, 191, 50, 102, 68, 318, 115, 111, 35, 234, 316, 216, 215, 39, 252, 141, 11, 106, 189, 121, 309, 14, 19, 26, 38, 40, 43, 55, 56, 62, 16, 366, 288, 359, 333, 301, 87, 249, 206, 126, 345, 247, 185, 162, 158, 258, 54, 8, 4, 28, 72, 220, 123, 75, 226, 367, 350, 1, 208, 7, 41, 99, 280, 137, 31, 151, 17, 264, 330, 153, 48, 37, 88, 255, 362, 323, 9, 116, 295, 222, 130, 70, 250, 160, 163, 29, 211, 200, 224, 147, 117, 341, 165, 373, 266, 328, 186, 302, 94, 42, 282, 90, 140, 322, 164, 84, 251, 339, 343, 169, 378, 149, 86, 143, 193, 173, 310, 66, 71, 73, 76, 101, 103, 105, 112, 119, 107, 218, 238, 133, 306, 256, 354, 290, 15, 262, 203, 312, 171, 178, 60, 246, 337, 311, 159, 196, 59, 61, 230, 64, 2, 174, 47, 327, 82, 33, 272, 150, 268, 91, 271, 157, 276, 297, 254, 95, 315, 30, 134, 346, 144, 285, 357, 181, 34, 24, 124, 93, 6, 78, 253, 18, 63, 291, 233, 286, 204, 100, 380, 265, 304, 129, 205, 139, 279, 202, 236, 336, 113, 361, 212, 132, 275, 22, 12, 358, 20, 44, 207, 152, 349, 244, 313, 122, 127, 154, 156, 235, 242, 257, 281, 303, 108, 259, 368, 248, 370, 231, 36, 287, 194, 321, 284, 214, 3, 292, 138, 237, 58, 89, 69, 300, 340, 161, 52, 135, 168, 51, 263, 299, 83, 228, 374, 261, 219, 372, 223, 172, 329, 45, 5, 334, 125, 269, 317, 25, 369, 320, 145, 240, 180, 98, 46, 347, 21, 307, 109, 335, 167, 371, 381, 183, 201, 352, 53, 274, 277, 243, 96, 197, 225, 273, 0, 182, 49, 166, 296, 213, 198, 10, 229, 128, 217, 177, 32, 57, 289, 375, 308, 325, 338, 351, 356, 363, 365, 376, 377, 383] + 35: [182, 204, 131, 368, 277, 133, 271, 357, 174, 234, 11, 129, 242, 361, 353, 93, 58, 346, 9, 255, 324, 279, 380, 170, 322, 0, 40, 258, 132, 141, 212, 23, 262, 35, 352, 159, 313, 55, 241, 69, 101, 173, 296, 260, 108, 5, 226, 154, 329, 196, 49, 163, 137, 12, 82, 114, 349, 13, 4, 155, 140, 227, 341, 42, 314, 24, 181, 343, 30, 6, 87, 238, 36, 107, 14, 355, 278, 297, 336, 85, 66, 220, 303, 10, 345, 194, 46, 146, 286, 33, 305, 338, 25, 169, 280, 332, 363, 379, 157, 44, 205, 43, 67, 243, 128, 319, 224, 78, 201, 172, 130, 83, 56, 247, 53, 265, 268, 28, 91, 211, 306, 301, 348, 153, 161, 235, 374, 41, 283, 311, 334, 375, 45, 256, 15, 290, 350, 272, 208, 68, 276, 1, 298, 156, 233, 219, 171, 127, 142, 240, 27, 38, 261, 60, 193, 126, 315, 19, 143, 179, 369, 383, 376, 327, 21, 359, 105, 284, 54, 113, 254, 304, 366, 335, 365, 333, 373, 165, 223, 184, 135, 120, 100, 209, 158, 364, 293, 294, 51, 16, 81, 176, 106, 79, 203, 74, 331, 50, 269, 186, 3, 70, 328, 266, 17, 197, 183, 378, 149, 20, 112, 138, 125, 377, 139, 134, 312, 7, 222, 190, 124, 95, 65, 291, 162, 249, 59, 18, 215, 90, 160, 77, 231, 360, 117, 382, 76, 289, 337, 246, 109, 72, 103, 295, 229, 221, 344, 116, 292, 253, 202, 75, 214, 189, 216, 325, 302, 372, 232, 200, 275, 191, 259, 230, 299, 330, 145, 244, 354, 236, 2, 89, 308, 317, 26, 370, 347, 119, 371, 188, 39, 37, 245, 210, 150, 228, 177, 185, 48, 281, 362, 316, 122, 252, 358, 104, 92, 309, 250, 273, 164, 326, 31, 257, 94, 225, 110, 102, 96, 136, 121, 111, 351, 274, 318, 288, 187, 151, 213, 207, 285, 206, 8, 267, 148, 34, 167, 217, 63, 99, 22, 52, 166, 88, 180, 71, 248, 97, 239, 98, 339, 73, 321, 270, 175, 287, 62, 282, 144, 57, 32, 152, 123, 192, 84, 29, 86, 147, 195, 320, 342, 198, 367, 118, 237, 64, 61, 80, 323, 381, 307, 178, 263, 356, 218, 300, 199, 264, 168, 47, 340, 115, 251, 310] + 36: [304, 155, 352, 5, 293, 193, 138, 97, 158, 365, 247, 210, 374, 220, 176, 199, 265, 212, 364, 81, 299, 95, 71, 356, 94, 233, 160, 266, 362, 117, 101, 65, 17, 282, 110, 116, 88, 188, 99, 26, 7, 15, 174, 231, 255, 150, 225, 10, 245, 129, 119, 251, 242, 109, 18, 21, 56, 295, 149, 259, 66, 337, 120, 241, 367, 72, 326, 157, 29, 335, 173, 368, 190, 64, 307, 89, 204, 183, 55, 334, 222, 172, 148, 239, 346, 37, 221, 107, 96, 115, 123, 124, 253, 298, 313, 330, 232, 291, 331, 280, 274, 122, 76, 340, 370, 238, 207, 211, 285, 215, 201, 373, 103, 75, 297, 92, 12, 179, 156, 121, 187, 20, 283, 226, 44, 350, 131, 102, 227, 267, 108, 182, 135, 74, 114, 154, 73, 351, 60, 339, 258, 31, 163, 180, 263, 281, 53, 296, 240, 301, 269, 132, 235, 306, 8, 316, 314, 376, 224, 347, 83, 34, 284, 145, 151, 289, 234, 52, 33, 358, 3, 270, 147, 127, 11, 68, 320, 79, 361, 51, 128, 104, 372, 143, 186, 16, 336, 375, 360, 170, 353, 355, 22, 310, 134, 380, 200, 61, 100, 194, 14, 319, 359, 45, 196, 153, 322, 177, 162, 69, 46, 349, 198, 19, 342, 98, 272, 191, 161, 23, 287, 311, 181, 67, 219, 357, 354, 213, 112, 230, 82, 38, 144, 189, 140, 125, 366, 9, 246, 6, 244, 32, 303, 93, 195, 371, 85, 345, 50, 382, 341, 276, 42, 118, 70, 59, 136, 254, 277, 47, 363, 130, 206, 159, 165, 228, 338, 203, 292, 54, 378, 111, 250, 305, 302, 261, 214, 43, 39, 113, 252, 290, 315, 126, 86, 377, 0, 58, 197, 178, 184, 332, 348, 309, 288, 323, 321, 275, 237, 260, 264, 328, 278, 175, 308, 343, 168, 164, 57, 248, 87, 30, 2, 344, 257, 333, 137, 141, 62, 209, 249, 78, 171, 49, 139, 105, 369, 300, 294, 383, 192, 229, 35, 41, 185, 216, 77, 256, 133, 271, 48, 36, 166, 63, 80, 91, 318, 262, 217, 273, 279, 167, 381, 28, 379, 317, 329, 4, 208, 90, 202, 205, 236, 27, 106, 142, 325, 24, 84, 13, 1, 40, 152, 324, 25, 146, 169, 223, 312, 286, 243, 268, 327, 218] + 37: [218, 127, 252, 272, 367, 279, 174, 0, 223, 151, 40, 355, 227, 34, 60, 124, 185, 302, 254, 148, 27, 175, 292, 114, 365, 343, 291, 21, 1, 349, 149, 48, 19, 345, 126, 121, 156, 87, 298, 161, 231, 105, 164, 64, 262, 115, 173, 299, 361, 360, 341, 138, 153, 251, 206, 77, 353, 331, 245, 256, 284, 295, 282, 320, 178, 321, 358, 275, 337, 232, 165, 215, 145, 82, 319, 324, 370, 134, 122, 7, 120, 92, 380, 26, 339, 84, 96, 172, 220, 226, 236, 239, 249, 265, 352, 374, 69, 125, 29, 376, 179, 228, 30, 271, 130, 194, 240, 95, 158, 24, 129, 234, 322, 93, 6, 38, 242, 340, 342, 364, 309, 140, 143, 381, 289, 62, 222, 167, 208, 221, 52, 195, 327, 22, 347, 382, 207, 83, 200, 46, 338, 314, 243, 325, 157, 66, 59, 15, 183, 13, 264, 197, 212, 335, 334, 363, 210, 109, 74, 136, 259, 116, 285, 123, 135, 258, 304, 354, 32, 155, 90, 89, 273, 307, 244, 253, 350, 286, 293, 250, 142, 75, 31, 263, 170, 186, 266, 219, 132, 377, 63, 70, 372, 131, 16, 237, 368, 287, 65, 36, 163, 25, 166, 51, 49, 188, 277, 311, 10, 5, 71, 47, 88, 17, 351, 235, 147, 14, 192, 261, 86, 344, 73, 301, 39, 203, 41, 317, 315, 102, 18, 211, 205, 150, 2, 180, 241, 290, 144, 362, 274, 176, 308, 190, 201, 141, 100, 189, 42, 375, 154, 79, 280, 313, 336, 107, 113, 198, 72, 181, 303, 104, 369, 333, 373, 111, 94, 20, 278, 76, 99, 225, 248, 214, 326, 23, 54, 230, 209, 383, 9, 247, 56, 269, 68, 8, 12, 53, 294, 366, 268, 182, 118, 328, 37, 305, 133, 58, 199, 117, 85, 306, 44, 233, 159, 3, 318, 191, 202, 267, 193, 81, 257, 160, 332, 171, 67, 152, 110, 184, 97, 204, 359, 45, 270, 128, 316, 168, 224, 61, 329, 356, 50, 378, 238, 146, 177, 187, 323, 108, 379, 283, 169, 217, 229, 371, 213, 312, 106, 255, 78, 162, 119, 296, 297, 91, 260, 281, 103, 11, 346, 80, 139, 112, 216, 300, 43, 357, 98, 55, 246, 4, 33, 196, 101, 330, 310, 35, 276, 137, 348, 288, 28, 57] + 38: [6, 284, 321, 369, 112, 178, 154, 208, 373, 256, 177, 94, 70, 116, 352, 194, 139, 241, 23, 334, 157, 175, 372, 106, 324, 73, 274, 357, 115, 215, 158, 344, 117, 238, 104, 318, 351, 240, 85, 278, 182, 181, 276, 119, 110, 226, 207, 263, 261, 35, 310, 10, 42, 15, 118, 107, 105, 233, 81, 37, 80, 59, 62, 34, 159, 205, 247, 113, 155, 141, 137, 317, 18, 214, 375, 38, 254, 24, 285, 28, 160, 134, 169, 323, 356, 46, 210, 330, 60, 61, 64, 127, 185, 191, 193, 195, 366, 348, 203, 246, 292, 355, 40, 383, 111, 27, 332, 100, 74, 108, 245, 130, 120, 255, 69, 92, 147, 349, 49, 306, 31, 145, 189, 88, 152, 343, 3, 378, 7, 320, 202, 168, 21, 72, 235, 54, 296, 43, 265, 328, 101, 126, 360, 367, 368, 258, 257, 29, 311, 75, 93, 333, 290, 44, 338, 225, 211, 151, 52, 342, 196, 269, 45, 2, 294, 359, 250, 271, 281, 91, 174, 217, 345, 121, 143, 183, 370, 1, 365, 14, 273, 302, 232, 32, 109, 13, 66, 222, 198, 237, 304, 314, 266, 354, 230, 262, 11, 156, 171, 135, 236, 319, 293, 267, 30, 197, 228, 180, 199, 299, 227, 132, 248, 97, 56, 76, 9, 26, 51, 212, 167, 275, 99, 129, 187, 327, 136, 286, 163, 218, 82, 128, 288, 253, 87, 336, 268, 102, 22, 77, 176, 57, 213, 123, 96, 282, 339, 122, 186, 307, 289, 161, 150, 287, 146, 200, 243, 209, 48, 358, 252, 297, 162, 382, 201, 234, 166, 361, 65, 264, 192, 220, 272, 312, 12, 83, 305, 291, 103, 98, 84, 216, 114, 184, 242, 325, 346, 350, 331, 124, 19, 179, 301, 341, 17, 50, 221, 337, 279, 316, 63, 125, 379, 219, 144, 280, 259, 206, 0, 190, 68, 142, 313, 309, 20, 5, 164, 249, 53, 55, 329, 170, 295, 86, 41, 16, 78, 362, 172, 260, 347, 33, 326, 8, 335, 89, 322, 165, 36, 364, 149, 277, 229, 95, 204, 298, 58, 25, 173, 140, 303, 283, 371, 47, 308, 90, 381, 4, 138, 270, 224, 223, 131, 244, 380, 239, 79, 67, 148, 39, 315, 153, 340, 71, 300, 188, 377, 251, 133, 231, 376, 353, 363, 374] + 39: [102, 100, 353, 336, 187, 297, 372, 56, 319, 112, 248, 322, 323, 141, 14, 258, 87, 93, 213, 151, 244, 221, 28, 281, 161, 356, 54, 363, 254, 208, 284, 5, 41, 172, 185, 84, 135, 154, 58, 95, 25, 139, 13, 226, 344, 291, 148, 124, 261, 107, 17, 16, 65, 230, 326, 146, 167, 309, 289, 30, 33, 66, 365, 39, 178, 266, 232, 376, 205, 2, 217, 145, 334, 229, 293, 90, 311, 122, 355, 20, 158, 321, 371, 382, 270, 119, 219, 330, 186, 27, 171, 111, 216, 184, 142, 350, 282, 236, 127, 110, 15, 170, 153, 223, 163, 383, 290, 260, 367, 7, 18, 129, 347, 333, 227, 203, 252, 242, 80, 200, 40, 101, 11, 303, 286, 169, 246, 214, 312, 339, 36, 253, 197, 245, 181, 277, 292, 251, 378, 285, 77, 196, 125, 79, 283, 206, 23, 233, 76, 194, 37, 373, 324, 173, 366, 123, 234, 201, 55, 329, 348, 50, 255, 190, 168, 45, 250, 263, 152, 143, 211, 331, 131, 209, 276, 231, 224, 308, 375, 117, 38, 150, 73, 115, 317, 301, 341, 327, 195, 136, 19, 4, 86, 352, 116, 274, 220, 368, 315, 177, 199, 359, 369, 118, 212, 155, 144, 62, 235, 351, 247, 377, 60, 164, 133, 298, 126, 380, 82, 360, 204, 113, 257, 81, 160, 32, 121, 104, 210, 147, 31, 69, 26, 71, 320, 70, 97, 192, 132, 162, 182, 49, 46, 103, 279, 337, 106, 239, 294, 105, 310, 159, 98, 267, 166, 91, 228, 271, 120, 176, 92, 52, 51, 10, 0, 342, 357, 259, 157, 78, 287, 193, 165, 137, 243, 85, 64, 130, 9, 189, 314, 265, 22, 198, 370, 89, 108, 240, 134, 96, 180, 191, 328, 280, 379, 1, 264, 306, 300, 295, 68, 307, 114, 53, 99, 61, 225, 75, 374, 140, 138, 88, 381, 156, 48, 362, 34, 275, 272, 183, 332, 59, 63, 67, 296, 218, 335, 202, 249, 325, 313, 128, 299, 364, 273, 305, 175, 8, 6, 268, 354, 35, 3, 109, 346, 43, 302, 74, 207, 358, 174, 215, 24, 21, 316, 47, 237, 188, 343, 349, 83, 29, 179, 42, 222, 338, 262, 94, 269, 149, 345, 72, 238, 44, 304, 12, 288, 241, 278, 57, 256, 318, 361, 340] + 40: [297, 59, 12, 249, 2, 132, 187, 275, 179, 51, 21, 13, 182, 45, 202, 148, 345, 135, 7, 163, 159, 282, 294, 81, 365, 158, 85, 102, 175, 286, 46, 355, 201, 146, 261, 291, 54, 254, 363, 177, 203, 15, 255, 86, 114, 57, 265, 193, 156, 16, 379, 373, 262, 230, 351, 95, 229, 224, 296, 28, 133, 119, 155, 326, 366, 24, 112, 323, 195, 302, 90, 221, 270, 103, 125, 364, 240, 318, 211, 304, 110, 38, 19, 61, 107, 142, 144, 128, 252, 320, 8, 105, 317, 58, 122, 171, 207, 136, 272, 216, 181, 244, 258, 55, 160, 303, 10, 260, 71, 238, 209, 48, 349, 290, 360, 121, 340, 368, 82, 108, 357, 215, 89, 369, 141, 256, 87, 250, 251, 184, 298, 17, 26, 353, 120, 213, 247, 130, 69, 293, 131, 126, 278, 5, 288, 65, 91, 151, 0, 29, 66, 167, 263, 44, 169, 324, 329, 41, 33, 267, 113, 189, 321, 378, 226, 352, 374, 310, 375, 191, 157, 32, 266, 273, 178, 168, 228, 49, 346, 25, 248, 313, 20, 94, 164, 306, 27, 217, 22, 9, 50, 115, 269, 162, 188, 180, 67, 166, 227, 129, 239, 232, 23, 274, 6, 347, 83, 137, 358, 237, 284, 64, 186, 333, 253, 327, 271, 53, 1, 205, 18, 341, 289, 73, 60, 283, 339, 305, 96, 370, 63, 285, 212, 194, 196, 145, 37, 208, 336, 35, 337, 77, 200, 246, 312, 197, 140, 331, 183, 382, 101, 176, 4, 134, 30, 348, 43, 206, 117, 165, 343, 311, 299, 56, 235, 123, 315, 34, 152, 356, 371, 42, 377, 99, 88, 39, 361, 344, 154, 335, 68, 149, 314, 62, 325, 147, 92, 170, 31, 100, 190, 334, 93, 281, 75, 338, 287, 79, 40, 150, 153, 80, 74, 143, 78, 104, 332, 301, 127, 172, 3, 76, 198, 185, 236, 319, 280, 192, 316, 359, 204, 372, 308, 322, 116, 220, 259, 219, 383, 367, 11, 98, 231, 234, 330, 268, 233, 36, 52, 264, 118, 225, 362, 223, 84, 174, 199, 139, 242, 70, 328, 47, 161, 307, 381, 97, 354, 277, 124, 276, 111, 72, 210, 222, 109, 173, 257, 309, 279, 245, 295, 14, 376, 138, 380, 300, 292, 243, 241, 218, 342, 106, 350, 214] + 41: [205, 318, 202, 111, 0, 337, 341, 43, 232, 76, 206, 249, 56, 86, 139, 19, 153, 13, 150, 183, 136, 330, 2, 87, 180, 103, 200, 188, 342, 218, 311, 95, 226, 349, 163, 189, 16, 45, 279, 227, 129, 146, 126, 271, 313, 211, 214, 116, 118, 260, 105, 21, 335, 161, 230, 97, 317, 282, 195, 181, 360, 54, 371, 5, 160, 290, 68, 208, 364, 162, 73, 72, 209, 378, 380, 40, 74, 9, 71, 280, 250, 33, 213, 149, 257, 184, 138, 94, 88, 26, 287, 361, 219, 277, 29, 141, 350, 79, 359, 264, 383, 112, 235, 3, 144, 168, 234, 166, 96, 84, 256, 12, 90, 261, 140, 286, 283, 27, 147, 154, 82, 276, 212, 7, 294, 25, 156, 122, 171, 307, 248, 80, 351, 177, 204, 182, 345, 145, 382, 300, 348, 1, 169, 123, 268, 332, 114, 203, 224, 117, 228, 225, 305, 296, 199, 4, 352, 134, 152, 121, 106, 243, 50, 344, 299, 298, 133, 308, 167, 278, 210, 253, 65, 131, 51, 41, 263, 221, 130, 186, 81, 78, 285, 6, 336, 100, 173, 85, 48, 52, 233, 288, 125, 273, 24, 262, 237, 185, 292, 135, 124, 338, 347, 254, 238, 8, 172, 217, 220, 142, 374, 379, 174, 107, 333, 35, 113, 269, 42, 66, 370, 322, 302, 316, 266, 61, 331, 310, 372, 75, 334, 194, 358, 55, 241, 34, 196, 289, 137, 30, 60, 190, 59, 14, 119, 381, 91, 187, 375, 159, 274, 258, 366, 367, 270, 315, 157, 321, 319, 301, 284, 15, 295, 115, 252, 69, 109, 62, 58, 22, 148, 297, 247, 328, 354, 17, 175, 325, 151, 83, 304, 293, 216, 10, 357, 132, 346, 281, 170, 306, 324, 329, 201, 44, 275, 28, 363, 368, 101, 242, 339, 191, 207, 49, 198, 223, 222, 77, 53, 158, 323, 356, 70, 165, 244, 98, 178, 327, 179, 355, 197, 37, 164, 143, 89, 57, 369, 251, 23, 320, 215, 340, 231, 127, 32, 67, 326, 120, 314, 236, 110, 259, 92, 128, 309, 108, 93, 245, 362, 291, 373, 47, 343, 272, 104, 176, 240, 239, 46, 20, 303, 246, 255, 18, 192, 193, 63, 11, 155, 229, 36, 265, 39, 38, 365, 377, 267, 99, 353, 102, 31, 64, 376, 312] + 42: [306, 77, 80, 238, 98, 87, 268, 251, 101, 138, 88, 303, 33, 197, 154, 41, 282, 209, 164, 150, 356, 151, 351, 285, 178, 92, 277, 311, 0, 248, 60, 217, 181, 244, 312, 3, 232, 247, 362, 284, 75, 34, 261, 134, 256, 76, 353, 78, 86, 71, 38, 348, 375, 149, 215, 155, 221, 349, 258, 168, 195, 345, 267, 146, 292, 31, 338, 355, 382, 250, 346, 334, 369, 69, 24, 141, 65, 46, 347, 198, 187, 339, 42, 211, 192, 174, 243, 121, 304, 177, 372, 153, 318, 14, 99, 270, 266, 200, 166, 118, 319, 37, 212, 373, 16, 272, 135, 202, 241, 378, 359, 120, 302, 161, 296, 263, 295, 321, 222, 20, 2, 226, 316, 32, 68, 317, 231, 350, 333, 184, 140, 145, 193, 147, 264, 260, 343, 139, 79, 47, 36, 220, 67, 119, 186, 246, 49, 299, 331, 172, 66, 45, 305, 207, 43, 210, 157, 254, 324, 22, 327, 381, 249, 276, 18, 6, 223, 183, 72, 180, 109, 191, 107, 62, 206, 19, 143, 162, 199, 136, 322, 205, 214, 28, 269, 340, 51, 208, 89, 370, 25, 286, 123, 111, 17, 21, 196, 114, 274, 55, 103, 291, 367, 364, 132, 320, 59, 275, 48, 283, 315, 52, 290, 167, 12, 148, 61, 236, 328, 281, 90, 131, 228, 293, 255, 179, 301, 230, 115, 122, 15, 1, 169, 83, 245, 84, 158, 156, 298, 97, 314, 23, 240, 100, 288, 227, 354, 85, 5, 377, 7, 262, 70, 329, 294, 40, 335, 27, 332, 313, 159, 330, 357, 371, 323, 173, 365, 4, 225, 194, 383, 94, 219, 128, 253, 361, 229, 363, 74, 190, 142, 224, 113, 126, 216, 242, 58, 50, 176, 325, 189, 204, 358, 93, 279, 336, 300, 56, 116, 265, 152, 308, 239, 129, 366, 380, 326, 280, 125, 309, 8, 117, 54, 102, 201, 175, 30, 218, 342, 112, 26, 259, 237, 130, 44, 287, 127, 144, 341, 344, 213, 11, 104, 170, 252, 368, 374, 379, 133, 124, 310, 29, 13, 352, 234, 171, 82, 297, 278, 35, 73, 96, 376, 57, 9, 307, 64, 289, 203, 235, 337, 10, 360, 257, 63, 110, 160, 91, 188, 105, 165, 53, 95, 185, 106, 137, 108, 233, 273, 81, 182, 271, 163, 39] + 43: [315, 202, 211, 243, 61, 55, 252, 218, 319, 366, 63, 324, 364, 258, 209, 170, 200, 27, 30, 156, 80, 356, 338, 46, 318, 379, 335, 341, 16, 149, 342, 267, 183, 371, 1, 306, 115, 72, 368, 163, 232, 269, 271, 193, 13, 70, 185, 109, 255, 157, 137, 189, 249, 215, 199, 321, 37, 327, 317, 314, 212, 8, 130, 162, 350, 284, 105, 268, 178, 247, 145, 295, 133, 78, 230, 229, 112, 45, 289, 91, 228, 296, 15, 245, 14, 43, 181, 51, 195, 370, 58, 29, 340, 104, 294, 21, 346, 36, 240, 239, 219, 272, 69, 194, 74, 108, 172, 92, 159, 84, 52, 152, 125, 190, 337, 120, 56, 191, 127, 339, 288, 363, 86, 333, 220, 374, 347, 77, 186, 330, 40, 201, 168, 221, 282, 93, 71, 205, 32, 147, 382, 94, 122, 65, 180, 278, 351, 276, 54, 286, 287, 131, 277, 301, 124, 355, 250, 33, 44, 113, 357, 312, 305, 31, 6, 102, 136, 349, 372, 187, 41, 118, 150, 353, 3, 90, 334, 241, 236, 231, 177, 248, 169, 206, 10, 89, 320, 7, 4, 208, 106, 281, 283, 75, 198, 97, 101, 161, 167, 298, 256, 76, 358, 238, 100, 224, 260, 223, 166, 378, 142, 188, 323, 138, 9, 20, 22, 0, 336, 290, 375, 303, 73, 380, 49, 204, 48, 207, 275, 210, 251, 62, 225, 99, 151, 68, 300, 280, 291, 377, 265, 17, 88, 38, 164, 50, 365, 222, 39, 184, 226, 254, 141, 261, 66, 263, 270, 111, 18, 264, 135, 325, 128, 373, 82, 23, 19, 376, 197, 331, 5, 192, 126, 227, 361, 110, 292, 313, 213, 348, 176, 326, 60, 307, 328, 47, 310, 381, 116, 293, 53, 262, 107, 344, 175, 96, 237, 216, 257, 242, 235, 279, 196, 2, 345, 160, 311, 383, 146, 274, 26, 98, 203, 369, 103, 253, 85, 59, 123, 87, 297, 34, 343, 140, 83, 35, 332, 302, 67, 352, 121, 148, 308, 12, 299, 153, 42, 362, 259, 359, 179, 171, 316, 129, 309, 139, 273, 155, 134, 173, 360, 154, 143, 329, 24, 117, 367, 322, 285, 244, 95, 119, 144, 182, 79, 158, 246, 28, 57, 233, 165, 114, 11, 81, 266, 214, 25, 174, 64, 304, 234, 354, 132, 217] + 44: [64, 267, 95, 242, 26, 35, 275, 299, 334, 74, 351, 250, 63, 217, 318, 98, 124, 66, 212, 215, 238, 86, 173, 43, 24, 324, 199, 57, 13, 171, 85, 300, 101, 200, 330, 182, 251, 187, 342, 372, 383, 123, 87, 258, 349, 80, 79, 180, 41, 181, 273, 264, 190, 32, 142, 317, 155, 1, 226, 108, 214, 118, 147, 294, 46, 193, 241, 310, 346, 231, 53, 341, 283, 357, 78, 134, 316, 197, 308, 152, 23, 322, 65, 116, 245, 362, 309, 344, 107, 326, 230, 339, 71, 307, 198, 297, 102, 36, 350, 285, 280, 91, 61, 243, 239, 105, 186, 255, 347, 175, 207, 305, 237, 268, 93, 130, 295, 121, 89, 284, 289, 338, 221, 290, 356, 336, 371, 109, 379, 153, 192, 249, 50, 353, 164, 33, 172, 323, 111, 3, 113, 374, 159, 286, 304, 110, 281, 332, 203, 359, 28, 39, 122, 97, 358, 22, 62, 261, 277, 68, 329, 169, 115, 375, 0, 149, 47, 140, 382, 45, 380, 210, 206, 259, 225, 220, 302, 311, 42, 365, 364, 154, 208, 106, 335, 354, 148, 136, 373, 112, 248, 88, 15, 10, 369, 163, 84, 263, 227, 256, 265, 224, 73, 14, 82, 194, 272, 131, 287, 253, 271, 257, 137, 222, 177, 166, 178, 126, 76, 156, 117, 77, 188, 202, 99, 54, 355, 92, 44, 21, 19, 282, 59, 128, 201, 312, 157, 366, 67, 345, 25, 314, 90, 376, 160, 204, 141, 37, 298, 83, 327, 331, 360, 229, 240, 27, 370, 56, 30, 17, 31, 328, 120, 9, 7, 150, 367, 135, 361, 247, 170, 146, 168, 133, 319, 16, 320, 195, 138, 270, 363, 12, 5, 174, 321, 183, 211, 129, 274, 6, 213, 279, 254, 333, 266, 125, 51, 244, 218, 4, 38, 165, 296, 94, 276, 191, 292, 100, 75, 378, 96, 103, 381, 34, 352, 179, 340, 313, 348, 325, 269, 127, 48, 145, 233, 18, 236, 343, 143, 288, 235, 301, 306, 260, 234, 52, 151, 139, 189, 262, 278, 104, 158, 144, 69, 337, 223, 228, 2, 232, 114, 8, 40, 196, 81, 184, 209, 70, 11, 176, 303, 60, 185, 55, 20, 291, 246, 315, 162, 58, 368, 167, 219, 119, 216, 29, 72, 161, 252, 49, 293, 205, 377, 132] + 45: [131, 144, 79, 179, 116, 178, 369, 52, 165, 155, 208, 373, 367, 37, 275, 331, 284, 207, 253, 319, 239, 365, 315, 10, 212, 99, 242, 341, 87, 0, 216, 217, 63, 204, 105, 161, 292, 339, 147, 158, 160, 243, 177, 21, 47, 168, 140, 129, 143, 310, 92, 110, 317, 16, 151, 377, 68, 357, 353, 259, 43, 252, 370, 56, 127, 135, 75, 67, 66, 258, 24, 300, 14, 115, 227, 157, 294, 362, 64, 244, 206, 28, 272, 77, 214, 89, 69, 111, 86, 210, 53, 117, 11, 287, 326, 347, 283, 281, 61, 236, 186, 282, 222, 256, 356, 146, 298, 220, 333, 261, 141, 50, 273, 82, 162, 125, 29, 25, 274, 233, 95, 378, 269, 215, 246, 123, 325, 176, 73, 359, 80, 51, 191, 202, 180, 279, 7, 142, 104, 289, 219, 139, 91, 205, 185, 102, 84, 23, 45, 39, 156, 265, 306, 303, 361, 291, 344, 173, 35, 264, 374, 153, 198, 200, 88, 187, 245, 108, 240, 183, 169, 171, 49, 371, 368, 132, 133, 305, 3, 30, 211, 195, 13, 137, 336, 358, 270, 136, 22, 42, 337, 167, 350, 201, 109, 31, 372, 118, 286, 330, 189, 46, 145, 262, 249, 335, 277, 54, 383, 382, 166, 301, 360, 101, 20, 322, 238, 225, 128, 8, 100, 114, 41, 346, 295, 232, 152, 26, 154, 34, 323, 72, 113, 119, 188, 228, 106, 381, 320, 71, 15, 170, 33, 164, 199, 324, 235, 203, 351, 159, 379, 213, 268, 218, 121, 307, 328, 6, 94, 181, 74, 321, 355, 90, 76, 18, 55, 313, 376, 231, 380, 257, 288, 302, 338, 312, 9, 4, 263, 192, 112, 148, 297, 230, 38, 12, 174, 311, 96, 234, 175, 327, 254, 182, 48, 348, 260, 58, 27, 138, 62, 120, 329, 134, 32, 196, 280, 163, 78, 70, 366, 2, 296, 149, 375, 278, 247, 267, 97, 334, 250, 150, 223, 36, 81, 290, 122, 293, 59, 308, 318, 103, 343, 345, 332, 85, 354, 248, 285, 107, 363, 5, 340, 197, 224, 349, 314, 44, 316, 226, 251, 276, 1, 65, 309, 40, 342, 271, 352, 57, 124, 60, 304, 255, 229, 237, 241, 126, 190, 299, 184, 17, 364, 83, 266, 194, 130, 221, 193, 19, 209, 172, 93, 98] + 46: [207, 224, 247, 376, 339, 152, 102, 103, 199, 105, 253, 362, 2, 104, 353, 355, 49, 164, 342, 8, 93, 322, 99, 313, 71, 12, 348, 328, 4, 30, 245, 201, 337, 17, 372, 312, 329, 264, 299, 88, 163, 121, 144, 176, 26, 80, 357, 165, 255, 187, 156, 95, 22, 359, 24, 235, 331, 107, 340, 65, 73, 59, 287, 18, 333, 46, 365, 174, 252, 270, 237, 1, 202, 231, 225, 56, 150, 37, 300, 5, 53, 307, 294, 295, 220, 50, 184, 15, 43, 175, 132, 248, 189, 251, 293, 216, 106, 239, 170, 168, 374, 7, 68, 212, 278, 136, 232, 131, 324, 75, 23, 256, 368, 217, 76, 82, 161, 345, 311, 280, 38, 273, 208, 292, 25, 352, 284, 67, 360, 381, 323, 267, 33, 218, 114, 160, 10, 242, 258, 186, 169, 39, 66, 369, 120, 364, 153, 356, 283, 86, 233, 85, 90, 213, 183, 286, 110, 260, 244, 118, 228, 297, 146, 289, 214, 318, 14, 275, 277, 282, 335, 238, 316, 28, 63, 94, 122, 196, 173, 272, 227, 330, 92, 288, 358, 182, 281, 383, 147, 341, 361, 167, 44, 343, 58, 221, 204, 319, 139, 271, 268, 166, 126, 254, 193, 246, 51, 234, 141, 155, 180, 290, 87, 97, 64, 125, 188, 52, 327, 354, 137, 69, 96, 320, 171, 230, 162, 249, 62, 338, 29, 219, 91, 70, 21, 77, 203, 226, 344, 109, 261, 363, 380, 305, 298, 116, 133, 40, 296, 291, 172, 98, 366, 177, 123, 179, 222, 336, 191, 19, 83, 47, 31, 262, 151, 332, 72, 178, 13, 157, 108, 304, 112, 79, 20, 81, 27, 145, 370, 309, 200, 257, 16, 3, 350, 54, 347, 61, 142, 111, 303, 158, 45, 9, 194, 149, 210, 351, 198, 6, 276, 274, 34, 211, 367, 377, 206, 124, 301, 266, 205, 127, 334, 35, 263, 285, 129, 101, 259, 375, 265, 240, 229, 57, 223, 11, 314, 154, 130, 143, 36, 209, 215, 269, 84, 378, 134, 89, 159, 113, 192, 195, 241, 140, 349, 317, 128, 181, 321, 306, 243, 100, 346, 185, 115, 315, 74, 41, 42, 308, 373, 279, 190, 117, 382, 78, 138, 48, 119, 379, 60, 135, 32, 55, 148, 310, 326, 236, 0, 197, 250, 371, 325, 302] + 47: [54, 330, 346, 370, 296, 123, 136, 61, 93, 244, 367, 98, 137, 4, 267, 210, 231, 377, 53, 189, 322, 141, 24, 76, 203, 158, 0, 108, 361, 264, 250, 172, 74, 282, 62, 202, 20, 132, 331, 213, 295, 319, 177, 121, 276, 12, 269, 107, 360, 236, 316, 345, 257, 169, 148, 207, 59, 321, 31, 341, 356, 304, 363, 205, 113, 233, 225, 354, 379, 170, 273, 239, 358, 38, 68, 64, 71, 283, 325, 149, 144, 88, 100, 69, 40, 160, 135, 99, 79, 300, 84, 8, 167, 270, 21, 272, 162, 299, 65, 41, 131, 195, 87, 179, 27, 103, 118, 208, 129, 352, 366, 154, 196, 111, 280, 328, 308, 275, 156, 83, 117, 187, 259, 26, 66, 312, 81, 70, 306, 176, 218, 45, 17, 44, 193, 373, 298, 291, 101, 95, 30, 302, 333, 116, 229, 109, 120, 163, 368, 183, 279, 157, 194, 320, 228, 89, 153, 324, 309, 190, 182, 266, 353, 28, 287, 72, 274, 178, 351, 286, 336, 92, 339, 206, 261, 251, 104, 15, 348, 86, 256, 124, 161, 220, 248, 260, 152, 56, 150, 214, 318, 364, 265, 90, 243, 106, 200, 63, 168, 77, 204, 357, 175, 43, 285, 52, 301, 18, 119, 284, 372, 371, 134, 262, 376, 36, 232, 337, 329, 305, 155, 271, 294, 216, 211, 359, 35, 173, 192, 9, 201, 143, 238, 14, 16, 82, 315, 281, 34, 198, 164, 365, 221, 33, 277, 184, 226, 13, 181, 334, 254, 37, 11, 335, 102, 209, 23, 327, 217, 219, 252, 230, 222, 241, 186, 130, 258, 146, 245, 166, 355, 268, 313, 25, 311, 142, 383, 110, 344, 253, 115, 47, 246, 288, 96, 78, 381, 380, 29, 2, 127, 235, 3, 50, 375, 185, 347, 303, 42, 314, 48, 292, 67, 122, 140, 180, 94, 290, 75, 128, 240, 151, 224, 342, 73, 46, 374, 171, 165, 212, 51, 138, 105, 174, 133, 317, 307, 60, 139, 343, 297, 362, 97, 197, 215, 188, 278, 147, 80, 39, 85, 289, 1, 338, 234, 227, 223, 263, 369, 378, 91, 249, 57, 310, 247, 159, 255, 10, 22, 32, 382, 199, 242, 332, 237, 293, 49, 125, 19, 323, 55, 340, 126, 350, 6, 349, 114, 191, 326, 7, 5, 145, 58, 112] + 48: [271, 275, 86, 364, 311, 128, 112, 290, 323, 24, 317, 316, 370, 119, 346, 223, 102, 101, 3, 261, 123, 38, 146, 213, 56, 305, 45, 297, 263, 281, 21, 162, 247, 212, 330, 82, 23, 220, 148, 7, 218, 376, 358, 118, 87, 293, 195, 239, 219, 362, 133, 211, 347, 156, 47, 126, 322, 157, 135, 92, 52, 227, 280, 100, 17, 57, 69, 29, 278, 285, 93, 294, 97, 327, 15, 131, 203, 369, 299, 0, 252, 206, 303, 46, 288, 348, 310, 284, 245, 36, 201, 10, 260, 153, 105, 196, 141, 329, 344, 88, 77, 185, 58, 170, 33, 189, 12, 174, 242, 91, 117, 34, 276, 254, 19, 179, 314, 343, 132, 32, 363, 320, 382, 49, 205, 14, 76, 55, 120, 43, 269, 319, 198, 80, 90, 54, 255, 342, 253, 63, 4, 37, 155, 298, 274, 18, 190, 143, 20, 165, 228, 268, 187, 307, 53, 295, 64, 39, 312, 224, 334, 136, 129, 361, 25, 103, 259, 324, 61, 95, 338, 374, 231, 336, 345, 41, 67, 304, 70, 216, 184, 243, 339, 163, 351, 68, 200, 8, 332, 96, 188, 250, 207, 287, 318, 367, 321, 130, 35, 192, 151, 166, 107, 152, 159, 258, 235, 98, 335, 273, 48, 99, 127, 353, 81, 368, 357, 279, 51, 172, 74, 237, 350, 59, 106, 178, 13, 145, 208, 352, 296, 150, 138, 122, 286, 164, 371, 300, 337, 289, 137, 42, 169, 306, 373, 191, 199, 134, 31, 113, 173, 140, 246, 229, 232, 197, 50, 380, 341, 44, 282, 377, 315, 168, 379, 302, 222, 215, 301, 183, 251, 383, 167, 262, 79, 209, 267, 30, 40, 256, 331, 277, 160, 1, 204, 84, 308, 66, 65, 226, 116, 72, 26, 139, 264, 109, 73, 234, 115, 326, 108, 325, 272, 291, 161, 283, 233, 121, 75, 176, 154, 194, 28, 111, 158, 193, 360, 142, 144, 340, 180, 225, 202, 22, 328, 177, 365, 359, 244, 210, 240, 313, 230, 375, 147, 78, 125, 11, 9, 182, 16, 292, 236, 83, 241, 249, 5, 248, 104, 381, 2, 257, 186, 378, 6, 265, 85, 124, 214, 171, 94, 60, 89, 110, 266, 71, 333, 217, 175, 62, 356, 366, 309, 238, 114, 221, 354, 372, 355, 181, 270, 149, 27, 349] + 49: [210, 304, 248, 194, 338, 199, 56, 153, 190, 205, 351, 214, 216, 36, 150, 14, 286, 84, 169, 174, 251, 57, 299, 16, 311, 253, 300, 17, 316, 215, 314, 66, 262, 104, 27, 295, 333, 37, 163, 212, 32, 33, 122, 69, 183, 192, 259, 356, 328, 18, 227, 323, 234, 45, 282, 102, 350, 137, 22, 331, 334, 100, 355, 188, 107, 83, 75, 327, 201, 226, 191, 222, 4, 87, 24, 211, 48, 93, 329, 103, 128, 182, 318, 285, 324, 219, 112, 135, 383, 341, 291, 123, 129, 124, 145, 229, 302, 202, 116, 206, 78, 365, 271, 133, 312, 5, 265, 59, 91, 47, 151, 114, 173, 81, 198, 289, 359, 85, 221, 224, 284, 21, 26, 209, 19, 118, 270, 277, 362, 258, 168, 158, 360, 40, 138, 306, 46, 117, 371, 247, 35, 74, 278, 340, 354, 62, 171, 176, 166, 126, 301, 142, 9, 144, 319, 297, 266, 379, 187, 377, 134, 272, 42, 346, 313, 256, 60, 307, 96, 165, 31, 315, 373, 88, 8, 257, 357, 255, 106, 170, 13, 349, 317, 67, 217, 200, 10, 28, 131, 358, 51, 68, 208, 353, 120, 279, 233, 235, 86, 276, 197, 193, 240, 236, 148, 180, 111, 330, 23, 195, 361, 376, 20, 238, 189, 250, 110, 267, 298, 204, 54, 132, 181, 196, 275, 273, 53, 336, 155, 76, 225, 213, 268, 303, 310, 34, 136, 321, 274, 162, 159, 294, 146, 293, 65, 348, 11, 109, 140, 125, 30, 296, 108, 167, 325, 39, 249, 2, 366, 113, 139, 337, 364, 43, 243, 71, 105, 92, 152, 149, 231, 228, 130, 280, 64, 79, 119, 115, 177, 380, 309, 287, 0, 97, 3, 237, 347, 367, 374, 156, 220, 58, 292, 164, 261, 179, 94, 342, 363, 95, 61, 157, 223, 44, 369, 381, 141, 89, 77, 49, 264, 161, 339, 283, 172, 73, 6, 378, 345, 98, 288, 241, 185, 344, 322, 7, 368, 370, 320, 218, 239, 178, 15, 127, 263, 326, 281, 269, 335, 82, 50, 12, 242, 246, 41, 55, 63, 38, 72, 70, 175, 160, 52, 154, 186, 121, 343, 252, 207, 90, 290, 29, 254, 147, 305, 230, 184, 245, 143, 260, 25, 375, 244, 80, 1, 382, 332, 372, 352, 99, 101, 232, 203, 308] + 50: [306, 268, 180, 358, 353, 138, 315, 187, 252, 58, 90, 275, 6, 189, 214, 63, 185, 281, 256, 51, 28, 366, 240, 249, 126, 344, 153, 327, 152, 17, 77, 266, 148, 157, 350, 108, 135, 210, 217, 96, 119, 125, 294, 121, 39, 70, 134, 7, 372, 113, 21, 312, 136, 213, 177, 239, 324, 206, 375, 149, 225, 64, 98, 357, 142, 349, 11, 231, 53, 147, 154, 383, 258, 219, 259, 20, 293, 376, 137, 282, 196, 347, 2, 82, 292, 245, 99, 190, 61, 373, 341, 365, 103, 257, 264, 298, 72, 9, 175, 297, 193, 42, 86, 14, 380, 279, 69, 300, 270, 243, 160, 340, 38, 127, 333, 205, 3, 220, 336, 208, 322, 325, 156, 229, 159, 95, 194, 236, 226, 235, 10, 23, 116, 54, 146, 144, 355, 66, 269, 202, 18, 321, 287, 164, 227, 163, 295, 351, 224, 248, 359, 369, 247, 371, 263, 310, 307, 255, 317, 209, 102, 91, 334, 26, 356, 84, 139, 29, 85, 335, 151, 379, 131, 120, 40, 167, 221, 260, 276, 360, 198, 34, 179, 250, 106, 363, 211, 216, 207, 176, 237, 83, 33, 318, 129, 1, 343, 5, 75, 0, 370, 49, 15, 115, 232, 141, 41, 301, 16, 352, 31, 346, 284, 261, 267, 285, 186, 277, 242, 22, 45, 8, 299, 68, 158, 81, 74, 171, 36, 114, 234, 71, 378, 161, 46, 251, 184, 288, 165, 362, 24, 155, 174, 238, 118, 43, 289, 60, 32, 67, 339, 331, 145, 150, 323, 52, 253, 178, 241, 80, 78, 309, 183, 173, 228, 367, 55, 326, 222, 62, 204, 30, 280, 291, 332, 47, 330, 87, 181, 316, 37, 286, 278, 105, 308, 345, 4, 172, 192, 329, 27, 117, 348, 97, 218, 124, 338, 166, 123, 212, 57, 92, 188, 381, 169, 122, 311, 130, 12, 382, 88, 246, 361, 320, 197, 94, 314, 56, 110, 48, 162, 25, 100, 319, 215, 133, 143, 342, 271, 313, 223, 274, 354, 107, 104, 265, 328, 230, 44, 35, 89, 272, 377, 50, 112, 254, 109, 203, 302, 304, 283, 73, 195, 111, 305, 273, 191, 233, 244, 296, 262, 59, 182, 101, 168, 140, 368, 199, 303, 170, 374, 132, 201, 128, 337, 200, 65, 19, 79, 93, 13, 364, 76, 290] + 51: [24, 240, 253, 10, 114, 203, 342, 86, 130, 341, 201, 120, 98, 198, 100, 266, 210, 46, 279, 44, 149, 18, 190, 304, 103, 17, 81, 193, 351, 375, 88, 282, 292, 123, 325, 364, 66, 180, 363, 134, 36, 132, 113, 239, 52, 333, 286, 7, 185, 230, 222, 138, 261, 367, 56, 267, 252, 175, 191, 331, 303, 365, 50, 339, 274, 317, 366, 231, 354, 19, 93, 60, 377, 275, 147, 154, 55, 243, 347, 376, 21, 296, 370, 166, 116, 151, 181, 174, 352, 2, 61, 173, 179, 256, 269, 273, 137, 184, 194, 32, 177, 219, 260, 20, 335, 90, 359, 171, 234, 197, 322, 326, 82, 299, 320, 119, 232, 334, 225, 321, 74, 108, 270, 143, 62, 355, 382, 285, 371, 278, 12, 315, 254, 308, 360, 346, 152, 343, 76, 192, 217, 131, 136, 211, 238, 75, 300, 345, 205, 307, 37, 281, 196, 38, 358, 5, 73, 316, 224, 319, 268, 310, 244, 51, 54, 111, 57, 83, 305, 248, 6, 80, 361, 95, 218, 318, 124, 383, 290, 15, 117, 214, 178, 67, 168, 255, 349, 272, 70, 170, 156, 42, 356, 99, 110, 167, 271, 65, 25, 84, 141, 69, 118, 258, 204, 153, 3, 245, 183, 146, 209, 41, 340, 135, 353, 186, 287, 122, 140, 350, 373, 280, 298, 216, 314, 47, 372, 45, 344, 104, 295, 182, 306, 200, 302, 242, 97, 125, 379, 311, 263, 92, 284, 297, 208, 223, 91, 16, 378, 77, 337, 207, 381, 264, 30, 155, 226, 202, 8, 288, 9, 28, 206, 369, 43, 105, 257, 169, 236, 189, 237, 63, 291, 332, 309, 249, 112, 328, 29, 133, 228, 126, 176, 68, 250, 142, 265, 187, 87, 164, 368, 59, 31, 374, 329, 129, 195, 109, 220, 235, 229, 35, 312, 212, 289, 48, 246, 301, 330, 33, 357, 148, 102, 213, 40, 362, 157, 233, 324, 128, 39, 115, 26, 227, 294, 160, 262, 107, 251, 72, 127, 144, 22, 323, 188, 101, 380, 283, 259, 58, 338, 89, 13, 277, 139, 64, 34, 161, 165, 276, 247, 336, 106, 121, 163, 85, 221, 71, 199, 23, 0, 79, 327, 11, 145, 27, 49, 159, 53, 158, 172, 162, 1, 215, 78, 348, 14, 241, 4, 150, 293, 94, 96, 313] + 52: [332, 248, 151, 342, 379, 258, 217, 196, 311, 215, 116, 161, 23, 97, 33, 312, 74, 85, 138, 171, 358, 127, 84, 239, 369, 328, 272, 22, 357, 3, 63, 265, 285, 79, 50, 31, 10, 352, 5, 364, 243, 130, 334, 343, 365, 326, 61, 39, 223, 47, 208, 355, 216, 227, 149, 335, 9, 129, 295, 237, 49, 236, 318, 172, 145, 294, 362, 152, 14, 348, 120, 360, 29, 181, 259, 319, 113, 341, 253, 368, 12, 356, 309, 222, 278, 20, 315, 251, 156, 131, 142, 316, 324, 86, 158, 180, 26, 378, 219, 262, 381, 276, 75, 125, 346, 289, 192, 72, 80, 269, 136, 139, 293, 182, 167, 73, 211, 184, 176, 44, 339, 4, 189, 344, 117, 111, 197, 159, 210, 376, 257, 56, 382, 186, 225, 71, 140, 256, 366, 19, 286, 6, 213, 68, 350, 214, 271, 103, 370, 17, 108, 122, 240, 372, 162, 150, 338, 169, 306, 195, 299, 94, 147, 249, 16, 290, 209, 35, 106, 18, 54, 28, 52, 179, 46, 301, 45, 43, 373, 66, 87, 185, 42, 218, 38, 380, 203, 202, 135, 175, 200, 264, 107, 27, 252, 13, 228, 298, 323, 168, 246, 234, 104, 307, 283, 93, 177, 69, 230, 354, 198, 321, 163, 260, 367, 336, 188, 173, 233, 221, 70, 78, 277, 224, 141, 333, 118, 134, 146, 67, 11, 302, 95, 205, 137, 99, 109, 110, 287, 41, 270, 24, 297, 157, 58, 178, 121, 250, 112, 81, 37, 153, 275, 232, 62, 282, 292, 51, 76, 90, 363, 7, 34, 77, 89, 374, 349, 310, 82, 124, 280, 305, 199, 377, 226, 268, 238, 133, 204, 274, 220, 100, 115, 98, 353, 53, 279, 317, 206, 325, 193, 183, 102, 327, 300, 25, 263, 160, 114, 345, 30, 330, 337, 92, 261, 1, 88, 8, 229, 15, 143, 40, 288, 291, 347, 83, 187, 57, 59, 308, 32, 359, 64, 255, 281, 119, 296, 242, 165, 148, 273, 154, 128, 383, 191, 254, 91, 235, 2, 322, 320, 313, 55, 105, 164, 190, 361, 174, 266, 65, 166, 194, 207, 304, 123, 331, 241, 132, 340, 36, 244, 21, 267, 96, 231, 284, 371, 60, 245, 101, 351, 170, 155, 144, 247, 126, 329, 48, 314, 212, 0, 201, 303, 375] + 53: [308, 199, 350, 311, 136, 338, 70, 4, 196, 44, 335, 182, 346, 232, 5, 333, 360, 134, 219, 165, 368, 357, 214, 299, 273, 64, 233, 10, 246, 154, 192, 91, 197, 30, 104, 228, 251, 282, 303, 142, 34, 130, 224, 103, 98, 369, 222, 32, 47, 54, 159, 290, 327, 68, 276, 315, 101, 257, 53, 337, 234, 245, 58, 88, 31, 364, 263, 50, 258, 89, 137, 102, 131, 62, 380, 186, 348, 57, 185, 90, 316, 277, 138, 215, 286, 61, 106, 140, 342, 272, 264, 43, 113, 181, 289, 377, 367, 254, 110, 238, 247, 95, 175, 149, 157, 112, 164, 243, 132, 332, 86, 293, 261, 320, 329, 353, 180, 318, 170, 291, 194, 78, 63, 295, 317, 331, 306, 208, 92, 79, 195, 324, 35, 345, 21, 168, 207, 143, 158, 111, 227, 8, 326, 29, 176, 352, 212, 122, 206, 287, 28, 285, 27, 242, 370, 17, 179, 117, 42, 323, 204, 174, 148, 33, 312, 41, 59, 99, 105, 218, 220, 355, 74, 365, 256, 259, 366, 349, 252, 82, 22, 359, 343, 248, 307, 314, 141, 96, 249, 56, 84, 87, 126, 52, 127, 173, 152, 129, 76, 241, 268, 379, 15, 65, 340, 383, 374, 275, 73, 304, 100, 319, 371, 283, 69, 363, 213, 128, 231, 14, 341, 40, 60, 166, 297, 16, 202, 0, 236, 229, 119, 72, 271, 262, 221, 144, 274, 300, 235, 381, 183, 145, 12, 24, 325, 309, 269, 83, 351, 139, 13, 36, 230, 217, 120, 294, 153, 184, 361, 358, 260, 37, 305, 281, 382, 123, 321, 71, 48, 1, 171, 310, 301, 93, 167, 210, 376, 80, 298, 223, 49, 7, 177, 344, 267, 265, 188, 135, 121, 280, 189, 347, 198, 172, 66, 23, 200, 3, 38, 118, 239, 51, 250, 151, 203, 146, 46, 125, 226, 75, 336, 133, 107, 330, 266, 313, 284, 292, 237, 160, 26, 356, 201, 372, 187, 178, 150, 55, 156, 81, 11, 162, 279, 362, 2, 225, 255, 302, 288, 296, 18, 169, 9, 25, 39, 94, 124, 85, 116, 191, 97, 240, 20, 216, 373, 77, 209, 155, 67, 190, 378, 19, 161, 109, 6, 253, 322, 375, 354, 163, 211, 193, 45, 147, 328, 108, 270, 115, 278, 244, 334, 114, 339, 205] + 54: [363, 41, 299, 57, 44, 325, 154, 147, 370, 90, 192, 280, 167, 143, 300, 356, 64, 328, 181, 46, 286, 93, 258, 27, 371, 326, 71, 353, 265, 122, 121, 92, 246, 61, 140, 331, 112, 164, 327, 207, 189, 35, 367, 184, 171, 343, 145, 340, 84, 321, 241, 76, 212, 26, 166, 95, 236, 68, 333, 153, 210, 297, 172, 81, 319, 79, 217, 287, 18, 193, 48, 334, 359, 33, 183, 267, 75, 162, 223, 238, 60, 383, 229, 316, 110, 105, 195, 324, 168, 362, 10, 130, 36, 55, 78, 123, 294, 240, 80, 115, 158, 151, 53, 124, 125, 284, 282, 163, 173, 91, 226, 179, 312, 242, 188, 70, 62, 54, 174, 347, 103, 293, 40, 375, 186, 208, 329, 341, 25, 32, 245, 314, 269, 161, 4, 349, 137, 42, 235, 277, 200, 249, 348, 191, 380, 248, 37, 34, 364, 330, 317, 211, 22, 77, 87, 7, 65, 337, 72, 239, 378, 244, 69, 252, 104, 307, 221, 116, 264, 263, 372, 306, 232, 273, 342, 43, 308, 187, 270, 373, 291, 160, 227, 28, 114, 234, 368, 152, 144, 148, 169, 237, 214, 176, 117, 260, 296, 357, 82, 45, 16, 197, 3, 365, 98, 311, 156, 352, 199, 113, 275, 322, 215, 279, 290, 225, 301, 20, 129, 111, 29, 309, 196, 230, 323, 2, 58, 136, 128, 180, 170, 15, 202, 228, 5, 295, 213, 99, 283, 96, 355, 88, 142, 118, 254, 278, 231, 86, 259, 250, 13, 150, 304, 233, 182, 345, 155, 332, 203, 83, 120, 377, 14, 23, 206, 131, 379, 19, 109, 8, 101, 74, 205, 52, 177, 126, 354, 376, 303, 175, 298, 285, 141, 256, 0, 100, 222, 313, 133, 339, 17, 224, 209, 288, 272, 39, 276, 119, 139, 366, 11, 253, 262, 97, 85, 220, 271, 146, 6, 9, 257, 302, 201, 292, 178, 38, 149, 135, 381, 243, 24, 30, 216, 361, 351, 310, 56, 251, 198, 50, 47, 268, 281, 289, 157, 190, 132, 360, 63, 255, 305, 12, 108, 21, 134, 218, 49, 274, 344, 261, 335, 31, 67, 73, 127, 185, 138, 106, 369, 374, 318, 219, 159, 338, 89, 204, 94, 346, 51, 66, 59, 315, 194, 247, 102, 382, 320, 350, 165, 266, 1, 107, 358, 336] + 55: [154, 355, 304, 360, 248, 337, 270, 212, 354, 306, 57, 303, 262, 344, 322, 14, 349, 4, 218, 250, 140, 48, 267, 23, 113, 363, 266, 213, 98, 260, 90, 328, 216, 3, 246, 36, 268, 104, 125, 53, 41, 214, 180, 297, 101, 182, 45, 302, 335, 339, 277, 220, 67, 203, 175, 254, 235, 336, 258, 193, 0, 316, 89, 188, 201, 273, 230, 227, 359, 78, 145, 71, 242, 282, 186, 142, 88, 269, 47, 153, 276, 229, 85, 10, 77, 34, 109, 348, 321, 223, 173, 351, 25, 117, 236, 288, 247, 211, 138, 327, 179, 312, 161, 369, 174, 52, 35, 105, 341, 202, 50, 6, 80, 28, 198, 62, 382, 19, 334, 192, 74, 278, 189, 32, 366, 314, 243, 309, 238, 331, 353, 123, 22, 383, 54, 17, 340, 160, 2, 149, 228, 127, 146, 55, 12, 171, 205, 84, 167, 121, 298, 326, 151, 307, 30, 15, 280, 116, 103, 86, 126, 75, 18, 94, 332, 91, 239, 21, 333, 364, 31, 208, 232, 263, 283, 133, 285, 83, 108, 181, 69, 292, 59, 275, 241, 166, 191, 29, 362, 187, 87, 245, 82, 257, 357, 233, 271, 73, 16, 377, 60, 237, 38, 7, 1, 185, 365, 115, 107, 323, 70, 5, 284, 40, 122, 375, 261, 195, 172, 63, 373, 221, 158, 72, 200, 137, 176, 134, 20, 209, 286, 225, 338, 319, 293, 64, 240, 164, 141, 42, 162, 177, 93, 251, 124, 150, 253, 264, 294, 299, 368, 291, 129, 342, 231, 27, 147, 315, 222, 378, 281, 96, 165, 143, 300, 301, 99, 296, 56, 380, 170, 130, 379, 132, 120, 128, 183, 352, 51, 204, 305, 168, 356, 224, 148, 317, 135, 310, 136, 106, 287, 81, 39, 350, 265, 255, 295, 249, 210, 308, 234, 290, 110, 65, 311, 97, 371, 44, 156, 9, 100, 381, 159, 289, 11, 46, 76, 178, 226, 112, 370, 318, 313, 217, 68, 163, 252, 347, 197, 152, 358, 26, 272, 206, 325, 95, 102, 13, 194, 157, 37, 119, 330, 343, 58, 169, 324, 329, 199, 111, 361, 8, 79, 346, 114, 244, 259, 184, 118, 374, 219, 279, 256, 372, 274, 33, 24, 155, 196, 139, 320, 345, 49, 43, 376, 131, 66, 61, 190, 144, 207, 215, 92, 367] + 56: [321, 240, 156, 204, 9, 126, 154, 368, 251, 89, 83, 238, 324, 267, 144, 99, 215, 383, 23, 331, 82, 244, 284, 116, 74, 158, 201, 107, 236, 193, 289, 305, 380, 314, 111, 323, 290, 91, 224, 325, 365, 369, 248, 333, 319, 56, 5, 71, 336, 15, 122, 40, 282, 233, 67, 228, 344, 143, 159, 135, 355, 66, 378, 155, 312, 218, 176, 178, 115, 182, 3, 65, 257, 165, 85, 95, 77, 24, 43, 68, 136, 192, 124, 55, 382, 320, 239, 247, 151, 276, 25, 37, 50, 61, 90, 103, 53, 101, 342, 0, 181, 160, 299, 134, 338, 356, 21, 104, 148, 377, 343, 12, 361, 79, 29, 341, 123, 139, 315, 316, 186, 88, 52, 297, 140, 60, 220, 242, 269, 362, 149, 259, 198, 196, 232, 359, 141, 335, 185, 47, 8, 189, 379, 357, 349, 188, 175, 351, 332, 381, 372, 235, 153, 306, 350, 373, 352, 273, 46, 73, 227, 348, 1, 237, 367, 221, 265, 51, 78, 260, 206, 345, 2, 254, 255, 280, 31, 129, 86, 293, 358, 105, 110, 234, 145, 256, 92, 132, 302, 108, 109, 142, 226, 163, 327, 191, 138, 184, 207, 6, 281, 150, 245, 114, 295, 42, 303, 219, 33, 339, 328, 27, 363, 72, 317, 310, 283, 48, 225, 102, 261, 347, 253, 197, 147, 113, 376, 203, 230, 301, 18, 268, 64, 311, 169, 128, 194, 329, 208, 205, 272, 309, 318, 243, 69, 80, 59, 49, 171, 38, 13, 195, 250, 164, 222, 252, 63, 57, 217, 20, 35, 120, 271, 258, 246, 26, 286, 313, 340, 76, 45, 274, 170, 296, 127, 326, 360, 200, 152, 262, 173, 133, 112, 180, 364, 146, 202, 287, 75, 294, 11, 322, 187, 44, 166, 374, 81, 241, 162, 100, 212, 209, 337, 231, 10, 278, 117, 292, 210, 39, 106, 97, 84, 172, 177, 62, 334, 168, 28, 93, 96, 300, 174, 304, 275, 199, 279, 167, 14, 264, 30, 130, 34, 121, 308, 4, 353, 266, 249, 98, 119, 16, 298, 223, 291, 183, 161, 17, 125, 277, 32, 36, 213, 330, 19, 190, 94, 137, 354, 7, 375, 157, 214, 371, 346, 263, 229, 131, 87, 54, 22, 58, 216, 285, 179, 307, 270, 41, 70, 118, 211, 370, 288, 366] + 57: [254, 345, 302, 90, 342, 94, 50, 33, 366, 223, 227, 346, 180, 260, 40, 21, 150, 383, 238, 352, 211, 182, 102, 136, 47, 300, 267, 339, 259, 54, 70, 66, 209, 109, 146, 208, 286, 34, 277, 188, 113, 287, 25, 86, 92, 198, 91, 246, 273, 174, 266, 176, 149, 186, 295, 173, 151, 127, 175, 7, 160, 200, 330, 72, 237, 298, 310, 343, 17, 2, 335, 329, 46, 119, 309, 99, 142, 278, 171, 229, 271, 112, 202, 178, 107, 101, 361, 131, 241, 234, 24, 194, 215, 221, 258, 275, 231, 125, 306, 43, 169, 356, 29, 347, 251, 217, 55, 85, 362, 158, 235, 134, 137, 236, 95, 203, 111, 270, 332, 122, 65, 60, 35, 71, 16, 228, 195, 257, 358, 297, 219, 324, 22, 199, 374, 177, 214, 115, 382, 312, 1, 61, 156, 325, 349, 216, 172, 26, 184, 193, 11, 344, 185, 274, 110, 212, 120, 321, 39, 207, 328, 222, 100, 363, 284, 82, 282, 213, 187, 83, 78, 77, 48, 292, 296, 370, 351, 132, 18, 320, 98, 244, 245, 291, 340, 263, 80, 232, 45, 262, 293, 301, 104, 68, 159, 58, 117, 226, 315, 272, 27, 179, 314, 62, 350, 41, 289, 196, 290, 152, 144, 87, 252, 118, 154, 138, 128, 123, 379, 106, 357, 79, 368, 103, 57, 116, 20, 225, 153, 67, 8, 12, 0, 253, 377, 307, 316, 157, 265, 162, 375, 224, 294, 168, 32, 166, 230, 167, 28, 281, 218, 256, 318, 313, 96, 23, 233, 360, 371, 206, 140, 121, 38, 30, 37, 14, 88, 124, 81, 163, 56, 164, 63, 276, 255, 3, 249, 69, 210, 376, 4, 336, 5, 299, 248, 13, 261, 311, 89, 44, 141, 264, 220, 6, 9, 197, 303, 317, 365, 268, 84, 331, 31, 279, 326, 288, 323, 367, 161, 108, 145, 354, 170, 130, 73, 201, 148, 133, 243, 285, 338, 126, 240, 341, 64, 191, 204, 369, 42, 181, 378, 155, 165, 59, 139, 283, 205, 242, 381, 247, 334, 348, 359, 74, 250, 327, 190, 319, 355, 373, 76, 114, 15, 322, 135, 183, 10, 51, 93, 380, 192, 304, 53, 189, 239, 143, 269, 97, 36, 129, 305, 105, 52, 19, 280, 75, 49, 147, 337, 308, 372, 333, 353, 364] + 58: [220, 348, 116, 178, 368, 344, 221, 347, 365, 216, 210, 192, 194, 213, 314, 86, 288, 175, 297, 318, 54, 82, 320, 312, 76, 269, 93, 232, 128, 137, 350, 94, 304, 219, 245, 27, 198, 222, 302, 166, 196, 89, 48, 111, 44, 16, 343, 70, 90, 295, 139, 217, 241, 25, 281, 177, 351, 107, 130, 160, 91, 7, 354, 319, 324, 135, 228, 339, 381, 71, 174, 197, 164, 140, 294, 1, 29, 57, 224, 17, 252, 218, 65, 363, 284, 332, 310, 249, 8, 11, 23, 39, 51, 62, 109, 117, 126, 49, 56, 380, 258, 40, 99, 115, 366, 199, 257, 242, 136, 255, 373, 151, 103, 66, 227, 342, 383, 9, 67, 68, 37, 259, 195, 34, 378, 356, 74, 278, 334, 315, 105, 149, 3, 64, 120, 349, 20, 69, 223, 361, 83, 268, 52, 155, 79, 207, 317, 293, 289, 75, 276, 0, 270, 327, 185, 337, 15, 285, 84, 275, 325, 85, 247, 208, 10, 323, 370, 32, 169, 180, 88, 96, 179, 146, 272, 188, 45, 362, 209, 87, 147, 33, 95, 38, 296, 134, 158, 159, 165, 184, 204, 212, 98, 148, 305, 42, 31, 377, 298, 129, 211, 72, 104, 277, 311, 266, 357, 133, 238, 308, 333, 182, 274, 4, 122, 121, 119, 100, 152, 256, 322, 307, 181, 80, 35, 141, 286, 331, 59, 55, 14, 63, 283, 306, 187, 316, 113, 292, 6, 176, 355, 2, 153, 244, 60, 144, 251, 206, 19, 273, 336, 353, 47, 214, 170, 261, 24, 183, 112, 168, 280, 231, 142, 154, 157, 193, 260, 191, 326, 163, 230, 229, 330, 233, 201, 108, 371, 101, 162, 145, 150, 41, 340, 239, 240, 246, 267, 279, 22, 352, 28, 264, 124, 203, 12, 271, 18, 262, 114, 234, 102, 92, 190, 50, 250, 77, 127, 78, 58, 263, 341, 106, 205, 225, 46, 329, 236, 26, 13, 328, 374, 376, 382, 248, 291, 300, 321, 131, 143, 123, 172, 202, 36, 360, 358, 171, 21, 97, 30, 303, 167, 73, 237, 226, 282, 372, 156, 254, 110, 189, 138, 173, 61, 200, 379, 359, 161, 346, 118, 235, 43, 53, 367, 243, 81, 313, 265, 375, 132, 5, 290, 299, 287, 125, 186, 301, 253, 215, 369, 309, 335, 338, 345, 364] + 59: [288, 364, 59, 23, 274, 212, 258, 231, 266, 236, 224, 353, 377, 371, 113, 178, 115, 339, 378, 52, 202, 150, 91, 346, 260, 315, 36, 180, 187, 213, 314, 259, 262, 43, 372, 251, 107, 366, 312, 195, 298, 114, 307, 345, 174, 163, 222, 373, 3, 124, 65, 223, 38, 47, 352, 344, 234, 358, 317, 206, 84, 166, 102, 64, 216, 191, 147, 360, 263, 29, 96, 269, 280, 348, 83, 101, 293, 144, 276, 318, 14, 201, 249, 322, 215, 341, 253, 205, 31, 155, 4, 15, 27, 32, 67, 71, 75, 151, 97, 158, 300, 28, 148, 232, 182, 226, 184, 143, 278, 172, 159, 37, 95, 304, 53, 383, 175, 169, 60, 106, 250, 11, 70, 140, 285, 243, 76, 319, 248, 12, 120, 292, 246, 332, 183, 25, 185, 24, 26, 199, 194, 13, 306, 283, 61, 305, 20, 359, 130, 221, 48, 82, 146, 282, 19, 17, 121, 272, 382, 177, 370, 237, 116, 68, 5, 105, 77, 340, 367, 323, 176, 108, 336, 74, 168, 129, 2, 125, 153, 167, 119, 110, 165, 230, 302, 351, 173, 90, 111, 117, 171, 189, 327, 379, 81, 44, 164, 245, 320, 324, 329, 21, 34, 181, 295, 196, 86, 291, 338, 270, 197, 316, 290, 7, 331, 149, 252, 45, 241, 254, 350, 78, 356, 369, 209, 69, 57, 325, 265, 30, 160, 365, 16, 56, 161, 41, 103, 100, 257, 239, 233, 204, 281, 138, 35, 380, 362, 289, 85, 354, 39, 79, 337, 286, 145, 55, 333, 122, 88, 33, 247, 349, 328, 66, 287, 299, 242, 273, 132, 92, 303, 54, 208, 87, 368, 72, 179, 326, 228, 238, 49, 271, 128, 135, 190, 210, 218, 235, 296, 186, 46, 137, 73, 334, 203, 284, 10, 294, 42, 363, 240, 297, 94, 170, 198, 62, 256, 342, 109, 112, 330, 313, 376, 127, 214, 200, 162, 118, 142, 381, 98, 141, 374, 219, 131, 18, 347, 93, 311, 309, 279, 136, 152, 50, 343, 229, 133, 51, 63, 40, 275, 123, 0, 9, 255, 89, 104, 1, 8, 301, 277, 139, 6, 308, 261, 227, 80, 134, 157, 335, 192, 220, 267, 156, 264, 22, 126, 193, 217, 188, 225, 58, 321, 207, 211, 310, 244, 154, 355, 99, 357, 268, 361, 375] + 60: [323, 112, 71, 318, 6, 332, 102, 38, 283, 192, 40, 78, 152, 215, 237, 354, 234, 166, 133, 273, 106, 374, 95, 265, 342, 269, 115, 24, 58, 32, 144, 36, 268, 179, 165, 232, 352, 167, 271, 47, 278, 316, 114, 351, 27, 365, 148, 353, 126, 371, 96, 285, 211, 315, 20, 177, 101, 43, 90, 163, 361, 53, 307, 169, 231, 59, 131, 21, 184, 41, 360, 287, 176, 4, 327, 324, 196, 376, 256, 154, 336, 272, 348, 325, 340, 187, 328, 65, 274, 9, 13, 14, 31, 67, 68, 73, 313, 312, 46, 236, 85, 51, 57, 135, 155, 337, 130, 139, 84, 302, 212, 110, 60, 260, 362, 190, 188, 264, 98, 129, 244, 369, 55, 364, 174, 213, 296, 350, 137, 162, 291, 49, 151, 314, 266, 48, 333, 158, 214, 77, 42, 113, 334, 164, 346, 319, 259, 311, 282, 44, 141, 25, 80, 203, 276, 8, 226, 197, 372, 138, 17, 128, 299, 202, 229, 72, 12, 370, 368, 23, 383, 91, 250, 56, 228, 199, 382, 279, 380, 26, 89, 347, 377, 69, 275, 86, 109, 122, 125, 147, 157, 160, 349, 153, 297, 123, 335, 194, 263, 107, 172, 39, 100, 195, 83, 2, 378, 111, 3, 140, 124, 15, 356, 99, 189, 230, 121, 132, 52, 64, 247, 246, 82, 358, 262, 11, 191, 117, 338, 366, 239, 217, 252, 45, 286, 173, 175, 88, 331, 204, 227, 345, 233, 240, 241, 223, 290, 293, 367, 156, 225, 178, 18, 19, 359, 103, 238, 220, 330, 251, 357, 363, 308, 216, 119, 373, 161, 22, 289, 105, 326, 10, 339, 97, 70, 104, 63, 118, 62, 198, 5, 200, 317, 168, 201, 206, 207, 210, 181, 134, 81, 54, 75, 306, 1, 180, 28, 108, 295, 381, 92, 344, 243, 305, 7, 193, 146, 61, 304, 219, 379, 294, 149, 300, 74, 120, 185, 37, 322, 245, 267, 284, 303, 310, 257, 186, 209, 258, 143, 50, 171, 255, 35, 116, 343, 222, 218, 145, 66, 76, 170, 159, 94, 281, 142, 0, 183, 235, 16, 29, 298, 136, 270, 208, 261, 248, 33, 277, 150, 320, 309, 182, 205, 34, 79, 127, 253, 93, 221, 87, 242, 30, 224, 321, 249, 254, 280, 288, 292, 301, 329, 341, 355, 375] + 61: [73, 341, 101, 225, 192, 176, 21, 263, 72, 28, 51, 366, 74, 122, 309, 45, 335, 367, 195, 200, 182, 338, 186, 382, 136, 202, 205, 216, 271, 377, 348, 286, 238, 224, 218, 83, 142, 169, 346, 327, 47, 17, 177, 105, 121, 170, 295, 129, 100, 29, 11, 23, 25, 143, 196, 9, 296, 68, 318, 69, 154, 323, 16, 19, 155, 302, 369, 114, 81, 181, 93, 3, 375, 149, 276, 371, 148, 63, 10, 364, 171, 210, 208, 6, 259, 266, 242, 58, 41, 0, 240, 275, 67, 332, 96, 347, 197, 36, 184, 91, 127, 183, 289, 61, 168, 261, 372, 243, 135, 27, 298, 215, 294, 280, 92, 284, 125, 111, 248, 297, 314, 55, 207, 285, 232, 119, 5, 365, 324, 308, 85, 246, 326, 262, 151, 70, 150, 352, 330, 59, 65, 157, 7, 77, 62, 333, 118, 245, 337, 131, 373, 78, 376, 103, 204, 368, 138, 126, 354, 79, 76, 161, 301, 44, 305, 18, 137, 312, 160, 99, 175, 292, 48, 49, 37, 56, 89, 359, 340, 339, 12, 325, 278, 124, 283, 8, 281, 123, 353, 106, 267, 329, 221, 33, 230, 277, 270, 350, 322, 203, 256, 255, 4, 66, 201, 288, 2, 166, 24, 198, 80, 120, 98, 293, 174, 213, 54, 102, 82, 97, 32, 117, 172, 362, 90, 290, 43, 315, 113, 264, 193, 253, 185, 235, 158, 383, 269, 217, 194, 336, 282, 206, 233, 22, 291, 317, 252, 279, 360, 144, 303, 95, 31, 84, 38, 378, 307, 310, 46, 251, 132, 247, 64, 226, 351, 116, 141, 34, 134, 187, 52, 115, 164, 222, 287, 254, 381, 236, 163, 345, 50, 231, 152, 53, 331, 13, 319, 39, 86, 159, 380, 257, 379, 214, 212, 311, 328, 190, 272, 227, 344, 30, 179, 14, 299, 112, 145, 223, 355, 146, 361, 139, 237, 358, 349, 220, 273, 304, 60, 167, 42, 108, 234, 107, 268, 147, 75, 229, 104, 239, 370, 374, 241, 342, 219, 88, 191, 260, 20, 211, 110, 130, 173, 258, 209, 156, 249, 244, 87, 178, 228, 71, 199, 363, 165, 306, 343, 313, 274, 357, 189, 57, 188, 128, 109, 321, 162, 334, 40, 26, 356, 250, 265, 153, 1, 180, 35, 94, 140, 320, 300, 15, 133, 316] +layer_updates_per_iter: 0 +num_slots: 384 diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-moe-load-balancer-configs/moe_load_balancer_gen_ep16_slots384.yaml b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-moe-load-balancer-configs/moe_load_balancer_gen_ep16_slots384.yaml new file mode 100644 index 0000000000..474423374e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-moe-load-balancer-configs/moe_load_balancer_gen_ep16_slots384.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [165, 150, 177, 163, 29, 68, 126, 243, 156, 200, 208, 375, 244, 365, 192, 270, 263, 286, 340, 334, 250, 226, 288, 185, 111, 0, 235, 295, 80, 176, 349, 284, 56, 311, 337, 38, 257, 331, 59, 221, 237, 98, 115, 179, 164, 121, 88, 61, 378, 259, 348, 188, 242, 139, 282, 54, 233, 174, 148, 358, 172, 249, 369, 292, 67, 229, 206, 191, 251, 24, 168, 303, 166, 290, 162, 169, 62, 65, 276, 33, 34, 90, 28, 268, 85, 103, 269, 106, 127, 6, 42, 53, 357, 92, 339, 58, 256, 224, 155, 182, 122, 63, 37, 307, 196, 186, 248, 27, 52, 108, 116, 215, 93, 210, 64, 353, 285, 298, 8, 13, 74, 173, 86, 312, 51, 255, 318, 359, 43, 354, 44, 149, 32, 213, 19, 296, 14, 50, 96, 12, 319, 329, 344, 310, 222, 330, 383, 332, 347, 70, 278, 181, 313, 297, 300, 123, 66, 146, 140, 180, 267, 102, 294, 2, 381, 374, 291, 247, 356, 217, 119, 230, 104, 218, 209, 55, 131, 258, 261, 262, 377, 240, 178, 211, 326, 154, 305, 69, 194, 271, 273, 370, 236, 49, 322, 193, 30, 157, 144, 336, 101, 129, 94, 234, 91, 227, 275, 379, 82, 260, 316, 41, 231, 142, 228, 274, 245, 203, 71, 241, 204, 232, 280, 360, 11, 36, 112, 89, 87, 46, 75, 238, 328, 252, 338, 152, 239, 189, 281, 201, 371, 184, 345, 207, 81, 321, 3, 141, 105, 265, 138, 199, 223, 137, 363, 1, 382, 350, 161, 175, 367, 287, 325, 110, 225, 134, 84, 264, 26, 376, 333, 78, 283, 109, 7, 31, 133, 335, 366, 320, 299, 342, 212, 380, 125, 18, 130, 277, 254, 95, 151, 346, 153, 327, 272, 22, 117, 124, 170, 77, 308, 136, 372, 195, 197, 145, 40, 219, 9, 23, 355, 373, 135, 17, 302, 190, 35, 343, 293, 72, 107, 361, 114, 160, 5, 317, 216, 97, 73, 20, 352, 113, 301, 304, 128, 120, 279, 79, 159, 76, 21, 48, 198, 205, 341, 57, 368, 309, 39, 4, 246, 100, 147, 47, 15, 306, 220, 167, 323, 118, 45, 60, 351, 158, 202, 289, 16, 143, 183, 10, 324, 314, 187, 214, 99, 25, 83, 362, 253, 315, 266, 132, 364, 171] + 1: [163, 178, 126, 241, 41, 56, 25, 37, 301, 335, 53, 360, 102, 330, 65, 156, 26, 346, 344, 369, 151, 66, 304, 368, 319, 134, 85, 168, 270, 300, 46, 211, 285, 162, 47, 88, 98, 362, 14, 165, 87, 8, 232, 297, 160, 16, 258, 100, 365, 292, 235, 21, 366, 63, 243, 275, 274, 218, 206, 264, 205, 140, 286, 321, 186, 181, 240, 309, 193, 45, 144, 199, 42, 272, 1, 378, 192, 294, 320, 260, 78, 166, 176, 203, 189, 313, 318, 57, 172, 311, 314, 375, 207, 208, 354, 267, 201, 82, 80, 149, 325, 271, 185, 71, 266, 187, 228, 289, 76, 121, 326, 62, 290, 352, 103, 83, 135, 10, 257, 310, 174, 111, 106, 150, 0, 250, 383, 86, 317, 197, 219, 171, 61, 79, 2, 115, 11, 113, 54, 279, 183, 132, 341, 255, 22, 225, 118, 43, 70, 23, 24, 242, 125, 288, 282, 381, 298, 122, 17, 296, 51, 268, 226, 371, 253, 361, 157, 34, 137, 120, 18, 263, 328, 248, 233, 32, 216, 92, 223, 342, 101, 164, 107, 19, 230, 96, 119, 246, 188, 252, 339, 265, 180, 198, 31, 196, 75, 244, 224, 173, 281, 84, 114, 278, 236, 238, 338, 175, 284, 227, 333, 331, 367, 191, 276, 312, 158, 182, 308, 50, 210, 153, 351, 89, 234, 194, 143, 239, 147, 348, 20, 184, 293, 13, 262, 372, 90, 124, 337, 15, 277, 3, 141, 254, 6, 73, 195, 9, 74, 261, 4, 38, 133, 170, 139, 229, 315, 94, 353, 129, 316, 374, 117, 327, 382, 377, 81, 329, 39, 334, 138, 145, 99, 280, 213, 202, 231, 222, 161, 295, 350, 302, 49, 148, 217, 128, 159, 336, 91, 67, 380, 40, 179, 105, 59, 303, 146, 379, 93, 358, 237, 305, 249, 33, 324, 55, 12, 190, 209, 155, 349, 345, 72, 347, 251, 291, 5, 58, 29, 269, 212, 109, 116, 306, 256, 204, 364, 136, 36, 323, 77, 357, 221, 322, 359, 363, 169, 167, 44, 152, 123, 299, 283, 247, 245, 108, 52, 69, 95, 259, 370, 154, 35, 97, 356, 220, 340, 64, 307, 215, 177, 127, 200, 131, 30, 110, 273, 287, 355, 142, 214, 376, 60, 7, 332, 68, 48, 104, 27, 130, 373, 343, 28, 112] + 2: [143, 230, 5, 197, 111, 355, 26, 279, 196, 273, 68, 383, 10, 193, 142, 2, 36, 331, 95, 183, 234, 260, 337, 229, 248, 180, 81, 151, 322, 280, 324, 83, 204, 130, 203, 297, 46, 69, 372, 176, 77, 352, 329, 307, 37, 353, 282, 300, 53, 122, 221, 75, 84, 157, 22, 235, 311, 87, 27, 361, 308, 160, 154, 179, 134, 359, 216, 48, 113, 64, 228, 330, 43, 139, 137, 149, 101, 271, 220, 24, 323, 263, 50, 187, 241, 309, 39, 362, 281, 57, 243, 74, 182, 80, 326, 374, 132, 314, 1, 100, 191, 73, 31, 146, 61, 206, 210, 58, 253, 103, 38, 268, 4, 272, 99, 251, 227, 240, 339, 199, 112, 304, 298, 189, 166, 278, 51, 217, 79, 299, 327, 201, 41, 186, 49, 94, 275, 231, 292, 98, 117, 380, 32, 366, 108, 86, 259, 125, 284, 286, 45, 274, 294, 140, 188, 236, 65, 15, 305, 152, 195, 141, 226, 158, 378, 266, 316, 381, 313, 377, 343, 192, 153, 306, 123, 340, 342, 90, 269, 138, 332, 129, 256, 318, 35, 52, 291, 267, 283, 249, 270, 190, 215, 91, 102, 34, 350, 66, 325, 63, 209, 17, 67, 148, 265, 181, 312, 219, 211, 144, 302, 59, 382, 88, 287, 40, 89, 168, 28, 321, 173, 245, 105, 333, 159, 356, 126, 218, 16, 169, 12, 29, 334, 233, 54, 164, 72, 127, 119, 161, 194, 62, 14, 78, 97, 131, 368, 200, 174, 170, 92, 76, 349, 276, 364, 104, 346, 261, 71, 167, 177, 369, 237, 246, 93, 19, 357, 60, 110, 44, 133, 82, 6, 7, 262, 198, 360, 116, 9, 257, 367, 207, 107, 205, 222, 13, 208, 375, 115, 290, 341, 213, 109, 118, 42, 244, 30, 347, 55, 20, 145, 348, 252, 23, 239, 319, 288, 8, 310, 289, 301, 379, 223, 232, 351, 33, 155, 70, 172, 85, 303, 224, 293, 21, 171, 175, 121, 0, 25, 264, 370, 335, 345, 214, 202, 363, 3, 242, 96, 162, 185, 135, 184, 11, 285, 18, 156, 120, 163, 250, 106, 150, 47, 178, 371, 338, 147, 317, 354, 255, 295, 212, 225, 247, 124, 56, 296, 328, 165, 344, 320, 128, 277, 254, 136, 365, 238, 376, 373, 114, 336, 315, 358, 258] + 3: [66, 113, 237, 184, 91, 110, 206, 378, 88, 90, 124, 326, 5, 247, 335, 233, 146, 324, 278, 303, 311, 222, 301, 370, 34, 236, 169, 383, 115, 20, 356, 101, 285, 45, 159, 4, 244, 371, 332, 52, 9, 257, 198, 199, 292, 59, 47, 38, 310, 56, 150, 337, 170, 325, 316, 21, 84, 32, 155, 299, 112, 30, 196, 291, 373, 362, 372, 338, 147, 265, 33, 341, 260, 13, 318, 215, 120, 189, 195, 118, 109, 229, 345, 256, 105, 268, 286, 361, 85, 153, 379, 107, 360, 60, 347, 17, 377, 317, 151, 331, 102, 250, 296, 290, 364, 359, 243, 218, 197, 280, 366, 298, 15, 259, 283, 190, 191, 369, 69, 152, 365, 31, 62, 50, 201, 58, 77, 11, 87, 154, 374, 41, 235, 65, 255, 322, 249, 130, 28, 354, 242, 141, 308, 167, 319, 51, 97, 49, 26, 164, 212, 182, 202, 161, 351, 126, 380, 117, 44, 276, 232, 376, 86, 0, 140, 368, 23, 173, 39, 209, 208, 334, 139, 336, 267, 174, 29, 8, 277, 300, 217, 355, 125, 306, 128, 106, 293, 179, 251, 1, 75, 185, 22, 230, 192, 158, 137, 343, 122, 177, 240, 162, 203, 176, 103, 270, 214, 43, 305, 381, 309, 183, 323, 95, 89, 193, 79, 35, 131, 363, 42, 168, 37, 239, 12, 78, 262, 98, 200, 327, 82, 160, 281, 123, 68, 16, 156, 148, 127, 226, 302, 275, 225, 348, 6, 63, 227, 74, 25, 136, 288, 114, 108, 53, 307, 100, 96, 294, 346, 133, 93, 284, 143, 175, 40, 46, 19, 24, 180, 178, 211, 221, 64, 271, 99, 297, 224, 57, 48, 328, 187, 367, 72, 269, 245, 73, 205, 315, 18, 71, 350, 157, 213, 207, 10, 304, 223, 279, 295, 92, 274, 94, 266, 144, 353, 7, 132, 254, 258, 329, 241, 320, 27, 263, 171, 333, 145, 339, 2, 231, 36, 382, 289, 111, 181, 344, 135, 246, 216, 67, 83, 228, 116, 234, 330, 272, 163, 313, 14, 80, 252, 238, 342, 287, 119, 264, 142, 149, 204, 55, 129, 104, 248, 70, 61, 282, 134, 121, 253, 349, 3, 194, 314, 261, 81, 186, 358, 138, 219, 357, 172, 312, 375, 166, 273, 76, 340, 188, 165, 321, 220, 210, 54, 352] + 4: [343, 253, 139, 361, 49, 35, 183, 133, 379, 216, 158, 66, 206, 237, 355, 311, 380, 134, 266, 231, 338, 182, 277, 282, 331, 86, 325, 151, 339, 22, 283, 85, 98, 261, 284, 220, 121, 327, 127, 10, 162, 202, 363, 242, 90, 107, 305, 332, 3, 320, 45, 345, 275, 365, 251, 82, 0, 360, 359, 70, 80, 92, 303, 140, 268, 129, 123, 293, 212, 337, 336, 351, 271, 259, 257, 47, 372, 357, 312, 207, 335, 252, 33, 232, 25, 267, 51, 249, 109, 205, 362, 144, 89, 225, 104, 120, 315, 258, 88, 143, 197, 76, 160, 8, 260, 243, 330, 341, 108, 288, 292, 29, 146, 211, 97, 48, 102, 301, 119, 58, 214, 375, 172, 171, 155, 200, 196, 280, 110, 116, 114, 169, 229, 310, 309, 157, 180, 71, 378, 316, 238, 299, 67, 321, 209, 18, 250, 105, 342, 145, 57, 73, 177, 99, 168, 323, 269, 328, 84, 184, 289, 130, 126, 247, 245, 326, 192, 63, 296, 340, 350, 5, 36, 347, 32, 27, 7, 278, 290, 62, 132, 69, 370, 176, 295, 147, 255, 374, 248, 285, 2, 287, 371, 34, 219, 19, 348, 167, 61, 218, 161, 187, 221, 189, 60, 125, 137, 138, 230, 304, 156, 186, 191, 265, 154, 208, 23, 188, 24, 376, 302, 329, 77, 314, 344, 369, 42, 256, 149, 74, 322, 294, 26, 170, 111, 115, 353, 11, 373, 20, 15, 152, 300, 281, 136, 274, 246, 101, 333, 141, 244, 166, 297, 262, 100, 306, 175, 291, 223, 203, 163, 91, 382, 272, 95, 254, 222, 239, 165, 377, 4, 83, 217, 317, 264, 356, 352, 383, 135, 349, 103, 122, 31, 215, 55, 164, 195, 233, 226, 148, 199, 228, 6, 56, 318, 235, 198, 153, 54, 307, 14, 64, 367, 178, 234, 334, 213, 81, 1, 368, 174, 17, 319, 142, 124, 9, 241, 13, 106, 364, 68, 159, 276, 72, 93, 131, 240, 52, 346, 118, 201, 21, 112, 224, 190, 227, 87, 117, 12, 16, 308, 298, 113, 43, 28, 179, 358, 53, 185, 313, 354, 50, 263, 181, 40, 381, 173, 366, 270, 79, 194, 273, 37, 193, 150, 59, 41, 44, 204, 38, 324, 65, 75, 46, 94, 39, 78, 210, 286, 96, 279, 30, 236, 128] + 5: [61, 65, 211, 252, 198, 131, 59, 251, 153, 46, 51, 56, 241, 254, 344, 221, 378, 86, 347, 307, 294, 326, 286, 354, 213, 260, 30, 72, 200, 304, 171, 181, 26, 45, 312, 366, 341, 119, 291, 283, 18, 297, 120, 282, 273, 348, 321, 75, 128, 308, 265, 145, 331, 173, 155, 140, 24, 301, 274, 227, 63, 90, 191, 205, 342, 322, 126, 311, 226, 280, 170, 368, 214, 208, 60, 255, 192, 74, 222, 2, 258, 16, 380, 14, 300, 9, 80, 323, 129, 189, 133, 328, 310, 83, 96, 302, 355, 172, 178, 78, 352, 335, 8, 206, 143, 295, 144, 20, 138, 383, 77, 271, 156, 125, 101, 350, 10, 27, 127, 44, 190, 21, 47, 372, 17, 338, 4, 332, 38, 3, 104, 266, 19, 250, 330, 134, 235, 68, 316, 267, 336, 42, 225, 196, 249, 259, 81, 122, 212, 364, 66, 358, 108, 210, 228, 186, 230, 150, 275, 361, 71, 299, 373, 292, 36, 180, 278, 118, 84, 159, 1, 365, 130, 246, 31, 70, 69, 207, 35, 115, 362, 201, 48, 87, 103, 114, 92, 139, 202, 116, 112, 177, 296, 231, 98, 381, 279, 7, 233, 137, 319, 320, 33, 12, 253, 284, 136, 229, 363, 135, 58, 185, 146, 357, 82, 157, 194, 272, 160, 169, 102, 293, 182, 163, 369, 333, 195, 79, 343, 219, 281, 29, 349, 40, 187, 197, 340, 248, 367, 303, 217, 6, 5, 188, 351, 88, 209, 151, 67, 204, 28, 99, 382, 121, 237, 13, 359, 270, 32, 290, 73, 234, 167, 179, 346, 288, 256, 223, 215, 89, 22, 268, 174, 34, 218, 329, 106, 257, 334, 360, 374, 107, 148, 11, 85, 339, 110, 277, 276, 52, 154, 370, 41, 147, 327, 37, 264, 152, 315, 345, 239, 240, 149, 236, 62, 244, 224, 53, 97, 109, 142, 57, 100, 356, 306, 49, 132, 43, 371, 313, 318, 175, 54, 216, 39, 161, 325, 232, 113, 263, 95, 245, 285, 123, 164, 337, 105, 287, 141, 262, 379, 91, 25, 375, 238, 314, 377, 93, 353, 317, 289, 94, 376, 168, 298, 309, 158, 117, 176, 23, 305, 269, 165, 193, 64, 247, 124, 15, 184, 324, 220, 242, 183, 203, 76, 162, 55, 199, 261, 111, 243, 0, 166, 50] + 6: [97, 378, 283, 178, 234, 336, 190, 139, 220, 69, 60, 254, 105, 381, 357, 106, 47, 250, 171, 72, 153, 331, 247, 257, 142, 350, 232, 118, 259, 128, 317, 77, 154, 87, 86, 261, 346, 70, 244, 108, 303, 263, 192, 265, 252, 40, 90, 161, 308, 371, 318, 84, 239, 113, 278, 111, 219, 365, 242, 217, 81, 224, 272, 73, 347, 195, 313, 172, 344, 204, 315, 342, 322, 55, 173, 88, 319, 300, 109, 231, 64, 332, 110, 63, 329, 147, 349, 130, 62, 351, 7, 26, 20, 75, 325, 373, 76, 341, 175, 187, 188, 156, 185, 49, 282, 143, 262, 116, 362, 18, 215, 39, 203, 338, 302, 213, 314, 132, 71, 164, 274, 269, 67, 216, 57, 196, 176, 245, 304, 107, 58, 271, 243, 145, 193, 268, 285, 288, 348, 339, 91, 61, 149, 323, 376, 249, 229, 181, 100, 184, 151, 41, 68, 327, 157, 361, 82, 14, 48, 125, 134, 226, 256, 374, 19, 382, 275, 35, 32, 59, 326, 260, 368, 79, 301, 182, 101, 211, 10, 206, 4, 194, 199, 309, 364, 251, 266, 38, 29, 228, 135, 320, 305, 212, 28, 9, 94, 255, 370, 179, 311, 321, 15, 6, 276, 355, 168, 8, 379, 150, 17, 51, 78, 92, 353, 167, 53, 297, 306, 209, 375, 112, 287, 165, 369, 158, 221, 286, 43, 284, 122, 3, 16, 366, 290, 102, 89, 241, 170, 214, 343, 146, 267, 11, 13, 367, 289, 95, 115, 30, 144, 296, 104, 335, 126, 124, 42, 123, 328, 162, 222, 1, 273, 52, 174, 177, 160, 205, 36, 189, 155, 0, 230, 12, 383, 324, 207, 197, 316, 198, 298, 235, 152, 121, 238, 140, 2, 65, 258, 166, 27, 46, 359, 293, 129, 345, 236, 202, 133, 136, 237, 159, 148, 358, 96, 292, 337, 294, 131, 127, 191, 363, 253, 233, 295, 120, 307, 119, 291, 277, 352, 279, 44, 264, 137, 23, 138, 83, 299, 312, 360, 24, 183, 93, 340, 186, 240, 66, 5, 31, 163, 56, 34, 270, 380, 223, 114, 280, 210, 225, 248, 50, 334, 74, 80, 21, 201, 141, 200, 25, 281, 99, 103, 180, 45, 246, 208, 354, 377, 37, 169, 98, 22, 330, 372, 333, 54, 310, 85, 117, 356, 227, 218, 33] + 7: [327, 260, 48, 181, 164, 119, 250, 177, 98, 314, 32, 335, 310, 247, 345, 216, 316, 80, 39, 230, 205, 279, 135, 60, 7, 308, 280, 61, 302, 29, 339, 210, 62, 321, 358, 220, 141, 233, 149, 222, 106, 88, 228, 266, 138, 122, 357, 50, 274, 258, 342, 27, 281, 262, 337, 255, 213, 368, 204, 322, 152, 373, 22, 79, 75, 218, 128, 195, 290, 326, 365, 364, 49, 341, 28, 188, 131, 100, 168, 319, 170, 241, 16, 129, 58, 306, 309, 52, 268, 295, 96, 334, 198, 352, 145, 240, 292, 163, 187, 97, 256, 215, 285, 374, 224, 132, 26, 180, 380, 155, 65, 120, 151, 36, 111, 379, 332, 12, 355, 257, 175, 369, 176, 15, 94, 307, 17, 325, 219, 333, 140, 190, 6, 353, 146, 225, 35, 69, 173, 43, 299, 127, 199, 338, 182, 298, 350, 354, 114, 113, 349, 296, 221, 243, 311, 186, 102, 56, 227, 193, 320, 194, 231, 252, 359, 143, 282, 42, 275, 362, 239, 378, 328, 47, 217, 360, 21, 377, 167, 20, 83, 121, 301, 93, 184, 112, 1, 297, 346, 165, 34, 348, 192, 25, 133, 118, 329, 249, 74, 55, 235, 191, 206, 130, 200, 134, 323, 153, 115, 166, 123, 317, 57, 66, 236, 160, 263, 201, 108, 99, 370, 73, 125, 23, 284, 300, 234, 237, 273, 226, 232, 87, 126, 209, 254, 3, 95, 197, 376, 110, 64, 103, 361, 71, 77, 183, 278, 84, 356, 344, 259, 136, 211, 67, 251, 38, 351, 169, 10, 277, 304, 331, 154, 371, 41, 18, 336, 271, 0, 313, 4, 2, 383, 264, 214, 19, 293, 53, 283, 101, 366, 207, 59, 381, 45, 150, 30, 179, 375, 105, 162, 253, 244, 294, 289, 270, 117, 372, 189, 109, 5, 44, 40, 291, 269, 286, 312, 124, 242, 147, 156, 91, 142, 340, 157, 13, 104, 14, 267, 148, 70, 330, 89, 245, 107, 86, 196, 68, 37, 287, 367, 303, 246, 172, 223, 276, 229, 382, 272, 139, 174, 11, 82, 288, 81, 158, 24, 92, 137, 305, 8, 261, 203, 363, 85, 63, 315, 54, 144, 202, 265, 343, 238, 46, 212, 324, 248, 31, 208, 90, 9, 185, 178, 51, 161, 116, 318, 33, 72, 78, 159, 347, 76, 171] + 8: [360, 64, 157, 156, 374, 150, 261, 198, 196, 72, 42, 379, 259, 299, 378, 145, 287, 193, 77, 80, 289, 19, 100, 207, 253, 108, 337, 380, 86, 69, 147, 65, 368, 120, 177, 45, 25, 151, 307, 265, 236, 357, 324, 208, 138, 49, 127, 170, 290, 161, 13, 252, 328, 22, 83, 216, 137, 334, 2, 8, 73, 90, 136, 99, 260, 33, 24, 245, 343, 57, 296, 234, 359, 104, 224, 36, 316, 131, 75, 119, 165, 183, 233, 4, 116, 249, 266, 105, 295, 54, 271, 285, 363, 0, 55, 96, 5, 273, 7, 228, 255, 280, 172, 38, 121, 309, 300, 91, 217, 230, 310, 37, 325, 306, 315, 348, 354, 305, 274, 213, 67, 281, 106, 12, 251, 209, 128, 6, 231, 303, 48, 197, 254, 98, 184, 372, 144, 9, 369, 158, 270, 203, 210, 242, 31, 182, 338, 312, 30, 11, 181, 283, 179, 275, 141, 211, 59, 248, 297, 126, 68, 93, 323, 103, 269, 53, 344, 58, 135, 204, 292, 241, 95, 364, 166, 23, 92, 178, 164, 341, 383, 118, 352, 155, 143, 322, 134, 84, 319, 331, 301, 132, 21, 229, 218, 154, 81, 377, 311, 226, 240, 304, 192, 318, 187, 191, 257, 314, 79, 35, 153, 146, 123, 70, 82, 163, 367, 371, 238, 173, 186, 18, 346, 133, 330, 40, 320, 358, 101, 87, 110, 41, 339, 175, 185, 370, 329, 342, 124, 227, 264, 152, 214, 282, 276, 97, 39, 109, 298, 215, 194, 60, 335, 237, 373, 382, 85, 89, 277, 76, 26, 43, 222, 262, 239, 353, 250, 272, 122, 168, 195, 333, 202, 219, 50, 114, 206, 174, 149, 47, 235, 159, 326, 220, 180, 51, 256, 63, 201, 308, 279, 381, 291, 349, 225, 34, 258, 112, 317, 52, 117, 212, 32, 190, 200, 205, 111, 351, 160, 286, 288, 376, 362, 221, 3, 88, 74, 115, 148, 188, 169, 167, 130, 125, 263, 28, 350, 327, 56, 375, 365, 15, 340, 94, 139, 1, 162, 62, 246, 243, 223, 44, 107, 27, 78, 278, 113, 66, 61, 232, 71, 16, 140, 14, 347, 129, 244, 142, 361, 176, 321, 10, 302, 336, 20, 294, 366, 356, 17, 189, 267, 332, 284, 247, 293, 313, 199, 29, 46, 171, 268, 355, 345, 102] + 9: [92, 198, 329, 32, 285, 268, 0, 311, 4, 181, 202, 18, 117, 322, 71, 216, 160, 58, 149, 40, 61, 168, 173, 26, 304, 29, 380, 132, 367, 153, 54, 87, 267, 189, 343, 175, 245, 79, 286, 38, 290, 368, 278, 257, 182, 338, 234, 57, 255, 172, 381, 242, 14, 9, 363, 264, 97, 186, 124, 332, 276, 307, 197, 110, 310, 232, 86, 43, 63, 362, 27, 283, 349, 19, 351, 370, 121, 133, 377, 297, 314, 269, 150, 72, 308, 379, 213, 254, 305, 346, 170, 108, 119, 28, 122, 340, 327, 81, 252, 37, 159, 56, 317, 3, 334, 116, 323, 238, 39, 354, 148, 112, 301, 162, 293, 243, 93, 204, 24, 365, 355, 166, 299, 344, 287, 282, 151, 154, 318, 41, 105, 66, 104, 248, 275, 203, 12, 125, 326, 68, 73, 378, 126, 348, 31, 78, 260, 184, 376, 209, 312, 187, 277, 107, 35, 206, 5, 75, 8, 261, 2, 375, 48, 188, 366, 361, 91, 265, 226, 212, 215, 222, 138, 67, 258, 135, 88, 192, 142, 230, 288, 235, 50, 373, 371, 333, 221, 59, 225, 101, 200, 77, 94, 10, 372, 179, 127, 136, 131, 302, 228, 128, 195, 36, 171, 246, 65, 205, 253, 270, 21, 337, 325, 223, 273, 152, 263, 147, 280, 155, 217, 23, 98, 356, 306, 364, 256, 16, 106, 324, 60, 233, 291, 134, 199, 167, 176, 118, 164, 25, 295, 103, 196, 165, 146, 51, 22, 158, 169, 183, 163, 11, 111, 13, 52, 300, 320, 76, 229, 374, 240, 279, 157, 7, 47, 95, 315, 219, 201, 341, 102, 30, 292, 247, 129, 336, 237, 319, 382, 174, 115, 330, 178, 335, 266, 328, 250, 156, 185, 244, 96, 74, 83, 271, 309, 144, 33, 321, 145, 70, 218, 208, 100, 190, 114, 251, 347, 34, 191, 44, 194, 236, 313, 64, 143, 357, 15, 214, 177, 82, 120, 180, 20, 207, 359, 80, 109, 45, 42, 224, 211, 193, 383, 284, 85, 369, 316, 49, 220, 350, 1, 210, 239, 161, 137, 231, 99, 139, 141, 339, 303, 62, 358, 352, 227, 69, 140, 296, 113, 53, 90, 262, 331, 353, 272, 281, 342, 360, 289, 6, 259, 123, 274, 17, 130, 46, 84, 298, 89, 241, 345, 249, 294, 55] + 10: [319, 263, 196, 76, 345, 360, 149, 233, 296, 9, 37, 267, 140, 13, 209, 111, 322, 251, 84, 156, 238, 283, 297, 369, 5, 8, 40, 154, 368, 208, 271, 229, 288, 219, 379, 97, 254, 220, 257, 206, 147, 118, 157, 242, 23, 295, 29, 336, 300, 215, 114, 47, 293, 274, 38, 32, 129, 75, 256, 230, 12, 50, 282, 287, 355, 158, 120, 143, 260, 82, 358, 68, 320, 126, 22, 351, 211, 312, 284, 212, 155, 214, 81, 357, 51, 125, 239, 113, 375, 373, 200, 273, 150, 228, 290, 366, 365, 137, 167, 144, 153, 304, 353, 356, 189, 266, 72, 311, 197, 25, 247, 323, 343, 169, 173, 205, 334, 347, 270, 291, 194, 268, 279, 101, 161, 19, 213, 234, 204, 218, 145, 105, 142, 327, 109, 64, 185, 307, 305, 303, 104, 172, 188, 324, 264, 93, 337, 225, 117, 138, 67, 272, 316, 249, 90, 341, 27, 112, 162, 354, 123, 96, 100, 4, 6, 344, 14, 381, 55, 246, 69, 57, 178, 66, 91, 252, 186, 61, 132, 21, 85, 328, 278, 184, 298, 65, 235, 35, 16, 207, 277, 349, 180, 255, 325, 79, 179, 241, 269, 363, 98, 231, 0, 87, 339, 378, 340, 174, 163, 159, 302, 165, 133, 333, 10, 70, 151, 192, 289, 102, 78, 330, 248, 136, 77, 376, 103, 107, 335, 338, 42, 39, 183, 131, 62, 17, 34, 352, 227, 236, 265, 294, 262, 73, 171, 210, 115, 11, 49, 359, 198, 232, 106, 41, 371, 275, 313, 226, 222, 217, 203, 191, 202, 110, 24, 301, 48, 2, 170, 99, 308, 240, 374, 285, 250, 56, 122, 74, 383, 317, 362, 299, 372, 224, 276, 361, 63, 89, 121, 310, 181, 309, 195, 237, 58, 130, 377, 382, 199, 135, 315, 20, 15, 332, 370, 364, 127, 86, 53, 83, 28, 253, 216, 146, 141, 92, 134, 182, 7, 243, 148, 306, 33, 321, 329, 26, 71, 45, 261, 326, 80, 292, 88, 281, 43, 166, 18, 164, 108, 367, 1, 124, 201, 348, 190, 223, 116, 46, 314, 59, 258, 193, 175, 244, 187, 119, 94, 44, 52, 176, 286, 31, 221, 331, 54, 177, 380, 259, 342, 280, 128, 3, 60, 346, 36, 318, 95, 350, 139, 168, 245, 30, 152, 160] + 11: [130, 376, 102, 217, 286, 154, 161, 69, 226, 89, 365, 327, 338, 302, 258, 138, 5, 112, 305, 175, 372, 155, 261, 269, 314, 260, 145, 144, 52, 136, 6, 45, 26, 10, 58, 336, 353, 311, 33, 182, 84, 210, 342, 216, 51, 212, 134, 335, 296, 77, 171, 22, 165, 146, 47, 213, 315, 316, 208, 219, 354, 324, 113, 7, 323, 222, 114, 328, 82, 329, 243, 241, 163, 252, 233, 345, 330, 177, 303, 367, 176, 133, 96, 294, 359, 118, 383, 19, 184, 4, 167, 344, 253, 334, 42, 12, 110, 156, 99, 0, 343, 381, 23, 170, 80, 151, 255, 242, 245, 193, 264, 355, 358, 306, 104, 71, 379, 122, 153, 61, 98, 15, 326, 218, 141, 282, 250, 280, 159, 374, 14, 209, 59, 158, 115, 225, 194, 271, 201, 272, 361, 111, 123, 105, 377, 285, 38, 332, 178, 337, 78, 371, 75, 56, 298, 270, 168, 179, 120, 62, 309, 37, 310, 183, 1, 202, 281, 109, 339, 357, 119, 24, 284, 318, 196, 124, 117, 251, 197, 81, 30, 135, 293, 263, 231, 274, 140, 132, 139, 53, 321, 191, 87, 126, 149, 162, 54, 204, 207, 320, 249, 27, 198, 259, 211, 279, 125, 142, 18, 108, 21, 268, 229, 333, 25, 256, 90, 100, 3, 29, 192, 70, 169, 297, 348, 347, 160, 41, 97, 292, 234, 174, 188, 106, 356, 277, 238, 239, 232, 307, 83, 248, 40, 267, 351, 128, 148, 313, 107, 93, 203, 186, 147, 364, 164, 57, 74, 103, 288, 304, 368, 276, 220, 273, 215, 44, 65, 199, 331, 95, 20, 346, 2, 172, 86, 31, 295, 378, 91, 64, 143, 200, 173, 380, 246, 181, 72, 257, 85, 116, 227, 300, 152, 39, 13, 66, 127, 92, 340, 101, 362, 34, 349, 79, 230, 375, 185, 68, 180, 189, 223, 50, 150, 36, 363, 157, 247, 131, 17, 94, 262, 373, 49, 289, 35, 366, 291, 43, 46, 360, 322, 214, 352, 317, 137, 308, 235, 11, 299, 190, 275, 244, 205, 319, 370, 195, 341, 254, 301, 265, 187, 48, 237, 266, 240, 278, 60, 28, 206, 9, 221, 121, 88, 325, 236, 73, 228, 283, 350, 166, 224, 55, 63, 312, 67, 129, 8, 382, 287, 290, 369, 16, 76, 32] + 12: [255, 288, 57, 291, 15, 350, 119, 249, 121, 257, 374, 252, 55, 72, 94, 100, 324, 73, 110, 272, 372, 220, 302, 42, 204, 292, 90, 259, 332, 368, 209, 107, 337, 283, 1, 45, 4, 38, 86, 297, 30, 282, 211, 269, 206, 64, 250, 225, 120, 109, 233, 354, 137, 25, 39, 322, 261, 286, 188, 165, 161, 11, 13, 244, 85, 375, 212, 79, 242, 213, 373, 53, 130, 175, 97, 208, 281, 317, 133, 171, 197, 144, 309, 0, 189, 169, 122, 112, 312, 310, 96, 236, 145, 314, 141, 366, 158, 344, 103, 260, 278, 330, 89, 265, 129, 157, 227, 52, 298, 319, 235, 336, 329, 47, 316, 320, 338, 328, 331, 199, 327, 49, 22, 151, 123, 216, 108, 102, 164, 23, 201, 114, 61, 325, 154, 379, 24, 383, 54, 125, 307, 166, 56, 162, 149, 363, 267, 50, 176, 205, 147, 153, 362, 326, 155, 200, 29, 280, 76, 20, 365, 274, 34, 81, 143, 360, 359, 222, 78, 156, 168, 192, 340, 68, 229, 27, 104, 80, 74, 253, 185, 67, 321, 355, 5, 186, 44, 93, 299, 221, 228, 66, 226, 341, 167, 290, 170, 382, 218, 364, 264, 63, 238, 370, 191, 62, 356, 36, 140, 217, 106, 113, 230, 7, 248, 43, 190, 46, 17, 124, 361, 195, 215, 60, 101, 135, 31, 357, 231, 138, 37, 159, 334, 263, 174, 131, 247, 111, 88, 239, 303, 203, 219, 126, 306, 295, 194, 87, 371, 273, 318, 376, 315, 256, 187, 270, 12, 128, 285, 18, 117, 118, 163, 377, 127, 333, 28, 349, 51, 6, 346, 16, 2, 241, 14, 304, 234, 224, 335, 115, 183, 207, 70, 178, 258, 184, 275, 240, 223, 172, 245, 289, 181, 3, 152, 10, 33, 308, 380, 77, 9, 313, 173, 40, 65, 279, 146, 343, 367, 26, 342, 353, 82, 150, 35, 71, 369, 339, 348, 323, 300, 271, 59, 142, 254, 262, 232, 305, 293, 148, 352, 347, 19, 182, 311, 99, 177, 243, 381, 296, 98, 134, 276, 266, 116, 180, 301, 193, 41, 84, 58, 214, 179, 284, 105, 345, 196, 210, 92, 21, 378, 69, 48, 287, 198, 75, 268, 277, 160, 294, 91, 358, 83, 136, 139, 351, 32, 246, 251, 8, 237, 132, 202, 95] + 13: [170, 46, 58, 40, 363, 147, 321, 333, 183, 267, 326, 245, 53, 73, 205, 336, 90, 20, 354, 155, 192, 204, 7, 175, 108, 0, 95, 17, 118, 198, 246, 320, 22, 239, 361, 222, 37, 220, 263, 94, 377, 285, 343, 152, 97, 368, 230, 32, 232, 96, 231, 88, 306, 369, 275, 208, 378, 145, 74, 52, 89, 379, 381, 154, 234, 233, 284, 258, 293, 156, 140, 316, 18, 81, 100, 11, 304, 75, 376, 334, 210, 26, 262, 221, 303, 345, 325, 308, 287, 119, 372, 224, 317, 148, 33, 78, 195, 77, 172, 134, 298, 45, 69, 229, 42, 179, 291, 256, 125, 337, 242, 113, 217, 61, 4, 60, 215, 47, 160, 165, 87, 137, 21, 102, 162, 295, 289, 158, 300, 237, 225, 177, 241, 68, 305, 133, 115, 365, 167, 203, 28, 269, 218, 206, 80, 194, 51, 340, 370, 176, 31, 143, 6, 355, 344, 121, 15, 34, 56, 257, 270, 213, 138, 1, 63, 272, 277, 301, 44, 9, 24, 209, 335, 279, 112, 91, 2, 211, 25, 196, 76, 223, 136, 244, 71, 124, 214, 101, 212, 341, 66, 193, 362, 318, 41, 254, 142, 323, 16, 261, 122, 79, 382, 93, 139, 288, 106, 259, 358, 313, 249, 117, 352, 290, 367, 126, 329, 274, 282, 330, 338, 131, 188, 294, 373, 219, 144, 359, 243, 36, 248, 127, 70, 235, 190, 85, 264, 332, 200, 315, 114, 30, 169, 276, 310, 35, 49, 297, 299, 65, 62, 166, 146, 105, 174, 268, 123, 13, 83, 157, 99, 347, 82, 314, 14, 12, 39, 201, 271, 159, 184, 273, 178, 98, 228, 107, 164, 331, 84, 92, 72, 38, 185, 346, 173, 251, 302, 350, 149, 238, 3, 342, 199, 171, 364, 356, 266, 10, 339, 253, 129, 163, 357, 348, 187, 5, 103, 383, 181, 371, 255, 324, 328, 189, 57, 110, 366, 307, 227, 54, 292, 375, 43, 374, 29, 48, 153, 296, 360, 19, 182, 180, 191, 236, 351, 27, 23, 280, 86, 55, 186, 116, 111, 309, 281, 168, 240, 283, 380, 8, 319, 50, 250, 130, 104, 226, 128, 349, 120, 247, 260, 312, 265, 135, 197, 109, 64, 202, 67, 207, 141, 151, 327, 161, 286, 59, 311, 150, 353, 252, 278, 132, 322, 216] + 14: [229, 228, 97, 182, 208, 246, 326, 133, 178, 377, 225, 264, 248, 173, 153, 46, 80, 185, 139, 51, 352, 205, 331, 165, 160, 383, 146, 3, 279, 166, 314, 201, 266, 109, 234, 0, 335, 63, 251, 195, 127, 281, 35, 84, 72, 29, 315, 174, 345, 274, 49, 254, 342, 271, 13, 328, 193, 346, 296, 380, 253, 313, 96, 273, 305, 190, 362, 236, 192, 38, 73, 50, 255, 56, 175, 99, 39, 250, 197, 343, 370, 382, 227, 105, 336, 12, 43, 262, 267, 18, 366, 247, 91, 340, 318, 88, 70, 147, 112, 23, 177, 374, 125, 158, 68, 104, 128, 101, 237, 242, 183, 376, 338, 74, 179, 293, 11, 219, 292, 260, 211, 275, 199, 142, 117, 360, 270, 33, 369, 333, 349, 48, 129, 223, 224, 42, 171, 78, 258, 55, 180, 134, 280, 67, 143, 71, 17, 235, 32, 269, 220, 59, 324, 301, 203, 222, 359, 202, 356, 114, 278, 136, 322, 57, 120, 252, 351, 289, 4, 155, 60, 348, 231, 21, 216, 149, 184, 108, 40, 156, 176, 89, 148, 285, 365, 238, 87, 132, 144, 243, 303, 98, 15, 308, 137, 337, 95, 79, 44, 135, 288, 363, 286, 82, 341, 152, 282, 106, 257, 20, 230, 62, 140, 283, 119, 83, 76, 378, 8, 118, 198, 61, 157, 206, 163, 339, 256, 5, 330, 168, 1, 311, 309, 304, 263, 31, 320, 27, 54, 196, 204, 355, 287, 353, 124, 162, 294, 215, 69, 37, 169, 347, 298, 300, 217, 28, 358, 334, 299, 194, 381, 368, 131, 272, 110, 126, 164, 150, 52, 154, 354, 161, 357, 22, 187, 116, 276, 200, 240, 214, 375, 361, 312, 7, 81, 307, 138, 212, 30, 327, 77, 186, 151, 213, 26, 379, 319, 159, 172, 226, 86, 115, 350, 310, 103, 36, 45, 367, 181, 65, 210, 249, 2, 122, 261, 344, 141, 371, 265, 323, 123, 167, 245, 373, 284, 316, 93, 6, 302, 9, 19, 92, 24, 188, 232, 295, 209, 34, 372, 268, 100, 317, 41, 25, 332, 145, 325, 241, 306, 189, 244, 10, 321, 14, 16, 47, 107, 113, 218, 121, 259, 75, 207, 239, 85, 64, 111, 297, 291, 277, 102, 90, 221, 191, 364, 94, 329, 290, 58, 66, 233, 170, 53, 130] + 15: [130, 236, 121, 23, 342, 235, 335, 246, 117, 282, 301, 180, 365, 123, 265, 91, 369, 280, 195, 125, 302, 192, 260, 85, 11, 185, 232, 353, 174, 18, 133, 372, 2, 50, 341, 347, 308, 169, 63, 190, 370, 136, 222, 304, 348, 25, 356, 328, 0, 39, 215, 157, 173, 138, 285, 209, 166, 34, 233, 339, 381, 45, 373, 218, 303, 149, 171, 330, 307, 19, 315, 276, 15, 199, 119, 329, 344, 111, 106, 5, 102, 77, 250, 382, 61, 55, 375, 54, 73, 14, 310, 228, 287, 219, 322, 368, 88, 291, 252, 221, 113, 143, 162, 323, 279, 10, 363, 30, 53, 204, 337, 52, 161, 29, 327, 181, 346, 186, 264, 270, 359, 4, 326, 51, 83, 241, 196, 47, 238, 248, 62, 12, 141, 244, 26, 172, 278, 126, 7, 98, 103, 249, 144, 93, 160, 290, 300, 203, 151, 277, 320, 36, 145, 269, 268, 13, 313, 314, 283, 334, 3, 188, 58, 361, 17, 297, 325, 281, 227, 298, 371, 284, 360, 193, 255, 225, 182, 212, 343, 81, 294, 72, 216, 95, 273, 197, 140, 124, 66, 120, 27, 46, 57, 68, 202, 333, 90, 376, 242, 379, 351, 247, 263, 150, 8, 191, 208, 383, 116, 65, 262, 112, 292, 70, 336, 366, 362, 134, 367, 74, 146, 139, 163, 31, 207, 105, 87, 229, 206, 349, 99, 37, 316, 274, 89, 176, 299, 21, 122, 127, 251, 104, 295, 266, 20, 56, 135, 115, 100, 305, 142, 96, 377, 321, 97, 224, 167, 239, 158, 187, 44, 110, 220, 6, 324, 148, 259, 159, 9, 16, 168, 24, 170, 49, 275, 69, 43, 107, 137, 64, 84, 79, 153, 293, 154, 243, 345, 75, 155, 175, 76, 231, 237, 35, 177, 109, 319, 42, 86, 22, 165, 226, 234, 71, 213, 147, 184, 318, 178, 286, 132, 152, 355, 217, 350, 332, 80, 48, 311, 352, 338, 101, 309, 331, 253, 306, 289, 211, 257, 210, 374, 380, 179, 108, 156, 131, 200, 128, 340, 38, 317, 296, 1, 230, 78, 28, 59, 205, 254, 67, 354, 92, 358, 378, 32, 201, 183, 288, 214, 198, 258, 94, 261, 189, 60, 40, 256, 164, 357, 194, 82, 114, 271, 118, 267, 272, 240, 364, 129, 312, 41, 223, 33, 245] + 16: [134, 274, 141, 226, 306, 377, 155, 8, 42, 339, 151, 200, 272, 91, 163, 20, 258, 288, 355, 192, 179, 94, 3, 148, 15, 224, 187, 16, 365, 248, 281, 305, 45, 137, 225, 57, 253, 109, 233, 268, 26, 176, 12, 74, 325, 98, 156, 249, 286, 5, 383, 270, 294, 97, 73, 182, 36, 9, 104, 60, 267, 142, 315, 17, 131, 241, 112, 130, 376, 344, 257, 353, 120, 330, 49, 374, 287, 212, 310, 38, 66, 219, 232, 22, 221, 158, 323, 196, 81, 128, 371, 352, 6, 135, 361, 111, 275, 67, 144, 183, 121, 123, 314, 335, 58, 41, 229, 238, 252, 208, 343, 207, 14, 127, 116, 332, 23, 40, 204, 126, 100, 68, 86, 205, 265, 87, 285, 307, 119, 184, 122, 160, 290, 102, 93, 48, 186, 113, 168, 149, 79, 254, 362, 247, 140, 162, 273, 35, 189, 347, 84, 51, 139, 231, 379, 297, 264, 18, 333, 266, 88, 157, 217, 150, 164, 329, 10, 345, 369, 11, 277, 259, 209, 367, 65, 2, 103, 31, 357, 322, 354, 54, 337, 30, 62, 284, 256, 245, 349, 341, 53, 13, 152, 271, 338, 348, 283, 246, 27, 115, 80, 380, 356, 276, 198, 132, 223, 190, 193, 324, 279, 61, 33, 291, 133, 69, 234, 75, 312, 129, 302, 364, 346, 52, 82, 153, 227, 372, 317, 29, 313, 244, 201, 47, 110, 138, 101, 326, 161, 215, 262, 280, 154, 342, 255, 46, 89, 237, 99, 282, 70, 202, 370, 340, 211, 105, 360, 107, 117, 210, 169, 197, 222, 309, 92, 118, 300, 90, 78, 235, 199, 71, 316, 39, 34, 293, 106, 228, 242, 378, 363, 145, 170, 230, 382, 263, 278, 366, 7, 303, 185, 203, 24, 236, 108, 328, 37, 159, 167, 334, 124, 336, 95, 304, 381, 218, 216, 243, 28, 350, 191, 239, 43, 44, 114, 318, 298, 213, 180, 177, 175, 0, 195, 301, 188, 375, 214, 55, 240, 261, 308, 251, 32, 299, 296, 368, 165, 64, 331, 289, 171, 295, 85, 321, 21, 56, 220, 19, 96, 25, 178, 319, 172, 166, 359, 260, 63, 136, 327, 1, 147, 269, 125, 143, 250, 146, 181, 59, 292, 311, 76, 174, 4, 358, 206, 83, 194, 373, 50, 77, 351, 173, 72, 320] + 17: [131, 197, 2, 75, 290, 177, 10, 272, 376, 36, 187, 221, 179, 249, 112, 124, 344, 227, 223, 285, 209, 169, 325, 354, 119, 193, 246, 370, 104, 381, 98, 298, 332, 20, 204, 377, 300, 262, 217, 192, 57, 132, 224, 306, 252, 14, 121, 152, 284, 91, 286, 8, 56, 180, 228, 282, 82, 13, 49, 115, 27, 299, 48, 274, 53, 280, 229, 71, 35, 159, 170, 202, 236, 208, 357, 266, 324, 172, 24, 38, 292, 304, 47, 118, 365, 44, 240, 287, 67, 318, 378, 257, 77, 51, 203, 235, 79, 80, 12, 232, 297, 207, 218, 273, 165, 191, 166, 215, 89, 181, 90, 329, 270, 31, 153, 349, 247, 111, 237, 251, 337, 307, 87, 5, 234, 339, 231, 143, 99, 162, 173, 41, 341, 265, 244, 64, 369, 65, 23, 109, 289, 268, 283, 312, 375, 138, 366, 275, 125, 167, 296, 243, 139, 314, 248, 42, 196, 271, 238, 183, 106, 350, 83, 141, 277, 334, 322, 320, 261, 212, 126, 225, 351, 175, 134, 199, 211, 214, 264, 61, 269, 253, 21, 361, 55, 308, 355, 149, 356, 348, 68, 86, 220, 28, 58, 359, 113, 328, 317, 335, 92, 74, 168, 331, 336, 309, 342, 102, 161, 364, 63, 241, 4, 178, 123, 293, 250, 374, 319, 155, 130, 62, 194, 73, 84, 256, 70, 60, 216, 382, 379, 310, 267, 372, 163, 40, 176, 353, 281, 122, 188, 362, 346, 117, 1, 276, 157, 206, 380, 263, 340, 19, 147, 101, 95, 258, 128, 158, 164, 230, 201, 0, 17, 127, 303, 294, 190, 136, 323, 189, 107, 148, 367, 33, 171, 43, 363, 186, 94, 302, 151, 278, 32, 315, 184, 242, 174, 34, 108, 305, 137, 311, 114, 347, 222, 97, 313, 9, 245, 260, 7, 37, 338, 120, 210, 76, 50, 316, 291, 160, 301, 358, 200, 330, 140, 254, 343, 6, 39, 46, 371, 135, 142, 116, 81, 198, 110, 93, 345, 255, 150, 352, 368, 288, 333, 69, 15, 26, 226, 219, 185, 133, 96, 22, 103, 146, 18, 45, 154, 205, 360, 195, 16, 326, 144, 88, 30, 11, 321, 72, 383, 100, 105, 78, 279, 29, 52, 145, 373, 327, 259, 59, 295, 3, 213, 239, 85, 25, 233, 182, 66, 129, 156, 54] + 18: [271, 374, 52, 74, 80, 97, 120, 156, 183, 222, 227, 242, 268, 293, 314, 344, 382, 1, 32, 59, 78, 147, 309, 340, 30, 95, 161, 305, 252, 336, 62, 134, 194, 221, 295, 310, 323, 196, 198, 353, 379, 49, 55, 91, 128, 186, 226, 327, 251, 315, 169, 139, 246, 99, 318, 20, 83, 316, 122, 328, 176, 319, 335, 153, 358, 66, 219, 369, 44, 299, 72, 206, 208, 333, 36, 355, 306, 341, 103, 371, 23, 79, 241, 250, 152, 178, 63, 292, 214, 48, 175, 229, 2, 258, 345, 11, 47, 53, 114, 136, 237, 330, 240, 188, 342, 87, 261, 7, 50, 195, 132, 329, 166, 108, 82, 163, 180, 372, 5, 100, 10, 45, 119, 189, 376, 35, 67, 164, 366, 121, 109, 232, 361, 54, 264, 13, 94, 383, 4, 51, 317, 85, 352, 173, 43, 193, 215, 236, 34, 378, 326, 70, 202, 262, 223, 296, 102, 111, 9, 267, 254, 303, 170, 197, 312, 42, 29, 18, 69, 272, 28, 142, 373, 144, 266, 46, 41, 104, 334, 288, 231, 125, 38, 145, 148, 127, 200, 224, 75, 107, 184, 117, 320, 356, 347, 116, 26, 151, 105, 274, 171, 359, 88, 367, 39, 8, 12, 363, 92, 179, 332, 17, 375, 90, 273, 160, 217, 118, 154, 380, 360, 297, 285, 269, 337, 149, 225, 159, 58, 140, 362, 351, 112, 275, 216, 126, 381, 115, 280, 212, 257, 6, 113, 192, 321, 291, 101, 19, 191, 73, 135, 350, 302, 205, 150, 182, 243, 213, 338, 130, 76, 84, 281, 277, 300, 260, 211, 174, 146, 357, 3, 204, 248, 177, 263, 22, 61, 203, 96, 187, 324, 256, 64, 259, 218, 86, 294, 290, 245, 185, 286, 239, 25, 284, 124, 98, 141, 343, 233, 278, 157, 349, 93, 255, 325, 348, 158, 311, 298, 172, 377, 21, 65, 106, 207, 110, 68, 282, 77, 89, 346, 283, 129, 0, 235, 308, 81, 155, 270, 60, 220, 56, 123, 354, 14, 230, 57, 228, 210, 368, 301, 253, 190, 181, 365, 331, 131, 137, 289, 15, 244, 40, 287, 234, 27, 247, 133, 370, 24, 249, 138, 71, 16, 304, 167, 31, 33, 165, 339, 307, 162, 201, 313, 279, 276, 265, 322, 143, 168, 209, 364, 37, 238, 199] + 19: [24, 216, 33, 117, 46, 127, 156, 2, 144, 336, 331, 280, 130, 82, 105, 361, 290, 367, 226, 220, 81, 98, 107, 133, 187, 70, 114, 349, 354, 25, 36, 337, 16, 67, 342, 237, 148, 88, 137, 208, 103, 381, 344, 282, 327, 198, 245, 253, 190, 91, 197, 38, 275, 147, 30, 182, 368, 347, 203, 213, 356, 376, 311, 132, 314, 106, 39, 86, 360, 382, 254, 263, 366, 52, 300, 164, 153, 171, 116, 165, 49, 297, 143, 177, 293, 281, 212, 14, 286, 357, 358, 163, 17, 43, 276, 355, 7, 48, 192, 184, 230, 292, 233, 310, 96, 40, 345, 346, 60, 372, 176, 139, 234, 112, 11, 21, 266, 274, 268, 319, 90, 339, 188, 193, 296, 9, 335, 6, 251, 257, 306, 284, 80, 3, 146, 87, 332, 150, 124, 51, 0, 55, 316, 369, 288, 243, 255, 373, 303, 189, 238, 134, 121, 272, 374, 85, 380, 140, 200, 186, 340, 231, 210, 100, 5, 78, 378, 375, 256, 141, 241, 8, 154, 271, 217, 246, 228, 353, 138, 31, 333, 23, 26, 326, 232, 174, 59, 267, 136, 56, 104, 93, 265, 115, 61, 183, 113, 301, 158, 13, 295, 68, 283, 110, 18, 131, 62, 329, 258, 235, 73, 269, 155, 236, 142, 109, 57, 307, 302, 318, 363, 97, 35, 53, 352, 172, 239, 249, 19, 108, 65, 362, 370, 247, 294, 289, 205, 248, 145, 229, 377, 64, 364, 279, 328, 89, 41, 15, 222, 270, 157, 4, 273, 160, 225, 341, 209, 28, 324, 123, 214, 261, 175, 305, 120, 240, 74, 219, 66, 159, 365, 285, 166, 173, 204, 215, 287, 75, 47, 119, 168, 63, 351, 162, 223, 308, 196, 317, 338, 101, 206, 126, 323, 152, 264, 122, 128, 167, 221, 259, 84, 202, 129, 321, 320, 94, 371, 178, 224, 379, 250, 350, 330, 312, 179, 322, 27, 77, 58, 54, 1, 83, 260, 277, 325, 262, 191, 383, 343, 149, 37, 118, 299, 102, 69, 12, 313, 207, 111, 291, 20, 334, 79, 71, 278, 76, 170, 244, 125, 45, 309, 242, 22, 181, 348, 185, 252, 151, 72, 34, 180, 42, 211, 161, 218, 227, 44, 135, 359, 29, 195, 95, 194, 10, 92, 298, 32, 99, 315, 201, 304, 169, 199, 50] + 20: [161, 309, 238, 20, 365, 253, 80, 142, 205, 72, 68, 103, 69, 27, 51, 52, 54, 61, 76, 107, 137, 167, 168, 179, 116, 111, 9, 49, 126, 194, 118, 247, 196, 248, 106, 312, 94, 11, 88, 341, 48, 70, 129, 109, 181, 189, 200, 206, 334, 339, 270, 324, 96, 292, 104, 364, 41, 213, 34, 169, 252, 326, 159, 151, 124, 354, 64, 147, 186, 143, 214, 262, 98, 14, 307, 318, 228, 187, 132, 315, 284, 175, 233, 114, 246, 33, 202, 230, 156, 321, 47, 207, 149, 222, 171, 263, 153, 272, 379, 188, 91, 362, 258, 182, 240, 219, 367, 131, 294, 78, 231, 351, 352, 28, 323, 313, 192, 275, 197, 266, 278, 174, 226, 185, 330, 229, 160, 358, 357, 348, 30, 150, 243, 338, 39, 24, 139, 199, 271, 136, 108, 301, 269, 281, 302, 37, 2, 105, 325, 95, 180, 172, 13, 35, 25, 22, 274, 232, 127, 327, 254, 296, 356, 157, 261, 306, 288, 299, 336, 122, 112, 353, 209, 273, 135, 121, 190, 378, 58, 208, 340, 310, 73, 89, 210, 289, 355, 211, 183, 344, 17, 241, 82, 8, 123, 32, 162, 290, 286, 77, 322, 343, 36, 256, 166, 5, 380, 191, 285, 79, 7, 225, 245, 31, 244, 303, 144, 282, 376, 204, 93, 311, 264, 65, 239, 375, 265, 372, 97, 297, 3, 212, 349, 115, 85, 305, 295, 152, 255, 331, 66, 46, 287, 62, 138, 125, 140, 320, 57, 235, 146, 119, 346, 12, 236, 360, 308, 319, 102, 83, 293, 155, 298, 350, 193, 335, 110, 291, 164, 382, 227, 184, 177, 267, 368, 178, 86, 370, 201, 337, 366, 250, 329, 276, 373, 203, 1, 242, 99, 4, 173, 195, 158, 141, 216, 92, 6, 361, 0, 18, 60, 100, 332, 374, 154, 283, 333, 16, 133, 220, 117, 249, 87, 145, 81, 363, 148, 128, 259, 101, 176, 328, 314, 279, 165, 377, 257, 75, 217, 21, 40, 19, 134, 251, 198, 260, 43, 268, 317, 130, 10, 221, 342, 71, 90, 218, 359, 300, 345, 29, 170, 84, 316, 120, 113, 63, 234, 23, 38, 277, 369, 45, 55, 224, 371, 381, 50, 347, 74, 26, 53, 42, 215, 15, 304, 223, 56, 59, 163, 67, 383, 237, 44, 280] + 21: [360, 92, 277, 91, 22, 366, 141, 234, 197, 64, 139, 347, 275, 160, 186, 333, 105, 143, 217, 265, 298, 297, 42, 17, 152, 82, 348, 118, 380, 25, 313, 89, 174, 223, 318, 378, 353, 337, 258, 273, 342, 156, 357, 285, 326, 311, 44, 35, 355, 46, 87, 345, 279, 349, 280, 288, 84, 309, 98, 125, 3, 14, 346, 379, 241, 253, 180, 54, 7, 231, 321, 339, 47, 236, 185, 45, 51, 128, 225, 220, 325, 331, 304, 159, 133, 301, 170, 48, 276, 163, 201, 55, 162, 32, 282, 350, 173, 256, 112, 120, 177, 365, 293, 221, 270, 49, 352, 312, 5, 30, 104, 126, 244, 181, 207, 85, 176, 59, 95, 58, 281, 190, 218, 214, 240, 147, 227, 367, 233, 267, 262, 97, 155, 154, 79, 203, 27, 110, 239, 99, 249, 61, 235, 93, 132, 153, 200, 232, 182, 151, 15, 9, 72, 372, 39, 383, 211, 40, 193, 216, 300, 315, 268, 114, 255, 116, 248, 123, 103, 289, 38, 361, 252, 184, 8, 111, 373, 73, 287, 286, 192, 266, 137, 274, 305, 195, 330, 375, 296, 166, 322, 264, 316, 302, 29, 88, 250, 351, 175, 144, 368, 63, 362, 303, 28, 237, 102, 329, 83, 334, 13, 130, 6, 31, 343, 306, 20, 140, 86, 169, 284, 191, 194, 230, 183, 229, 259, 260, 165, 295, 198, 269, 271, 319, 41, 374, 106, 142, 101, 108, 371, 113, 205, 199, 150, 26, 121, 52, 23, 1, 358, 21, 317, 100, 148, 340, 134, 381, 74, 2, 328, 188, 107, 179, 308, 257, 210, 117, 178, 307, 219, 209, 243, 131, 359, 66, 354, 115, 341, 57, 81, 382, 122, 68, 202, 145, 224, 369, 254, 71, 50, 363, 0, 16, 299, 43, 245, 338, 90, 251, 290, 65, 187, 261, 36, 12, 164, 37, 314, 212, 272, 10, 4, 119, 53, 94, 149, 172, 196, 135, 78, 263, 60, 246, 323, 77, 292, 335, 215, 67, 294, 56, 324, 222, 127, 327, 158, 376, 247, 364, 377, 320, 109, 344, 189, 18, 336, 356, 291, 171, 136, 24, 213, 310, 70, 76, 228, 161, 167, 332, 80, 124, 206, 208, 226, 238, 19, 69, 33, 168, 138, 62, 370, 204, 34, 146, 157, 96, 75, 129, 242, 278, 283, 11] + 22: [254, 58, 351, 375, 372, 122, 75, 321, 99, 341, 335, 241, 96, 262, 334, 342, 302, 43, 158, 231, 2, 28, 79, 95, 141, 38, 139, 55, 218, 312, 170, 228, 15, 165, 159, 256, 211, 136, 279, 52, 62, 344, 197, 163, 234, 101, 143, 154, 181, 332, 184, 93, 348, 7, 264, 259, 340, 49, 72, 301, 269, 268, 273, 103, 219, 64, 128, 162, 121, 297, 157, 217, 221, 126, 73, 329, 50, 338, 123, 226, 175, 324, 105, 303, 124, 45, 275, 108, 266, 291, 212, 23, 245, 313, 247, 250, 260, 9, 201, 244, 352, 111, 378, 27, 3, 25, 1, 305, 327, 286, 353, 232, 186, 350, 255, 326, 270, 315, 253, 267, 261, 204, 115, 169, 130, 345, 195, 60, 361, 191, 59, 39, 22, 373, 271, 176, 160, 36, 104, 363, 66, 167, 192, 307, 328, 33, 42, 263, 317, 149, 67, 125, 14, 48, 71, 320, 322, 32, 97, 26, 119, 30, 19, 354, 304, 243, 205, 310, 281, 316, 207, 112, 362, 153, 80, 83, 349, 113, 131, 133, 236, 189, 299, 356, 246, 137, 129, 296, 5, 277, 325, 346, 216, 371, 132, 74, 117, 272, 306, 8, 369, 82, 220, 239, 87, 377, 290, 364, 274, 151, 145, 308, 127, 295, 10, 249, 343, 94, 147, 4, 61, 357, 173, 76, 206, 358, 309, 41, 47, 78, 382, 46, 237, 92, 152, 31, 370, 166, 135, 280, 278, 110, 276, 177, 355, 70, 100, 161, 193, 339, 40, 367, 318, 11, 300, 379, 265, 120, 179, 150, 6, 293, 142, 287, 209, 376, 330, 37, 383, 381, 336, 68, 116, 156, 18, 210, 155, 54, 0, 314, 102, 196, 190, 223, 88, 107, 368, 359, 148, 178, 331, 144, 29, 240, 251, 365, 214, 53, 258, 380, 98, 283, 13, 202, 85, 182, 91, 168, 213, 84, 56, 289, 24, 20, 34, 230, 235, 229, 285, 222, 17, 366, 63, 12, 65, 227, 138, 215, 35, 187, 16, 69, 140, 114, 86, 292, 224, 183, 51, 164, 118, 333, 311, 174, 57, 347, 233, 225, 200, 199, 81, 77, 298, 294, 134, 90, 284, 172, 194, 323, 337, 288, 198, 242, 44, 252, 188, 106, 185, 109, 374, 319, 248, 238, 180, 89, 208, 282, 257, 171, 21, 146, 203, 360] + 23: [26, 374, 6, 309, 70, 152, 129, 119, 125, 341, 58, 28, 121, 25, 214, 9, 350, 22, 63, 186, 4, 320, 168, 288, 314, 197, 154, 218, 371, 184, 312, 194, 210, 66, 333, 303, 280, 108, 297, 355, 140, 88, 247, 199, 217, 189, 37, 265, 124, 77, 324, 180, 159, 340, 74, 317, 290, 157, 93, 137, 352, 200, 16, 285, 373, 282, 230, 334, 145, 262, 8, 286, 172, 211, 306, 232, 372, 130, 146, 151, 342, 370, 160, 178, 284, 256, 206, 236, 375, 310, 276, 259, 246, 296, 216, 81, 275, 376, 87, 176, 55, 92, 326, 229, 38, 359, 273, 347, 76, 109, 21, 349, 104, 188, 266, 171, 117, 13, 251, 64, 71, 69, 173, 47, 127, 257, 144, 149, 228, 44, 329, 207, 193, 339, 68, 362, 267, 367, 322, 181, 278, 18, 29, 242, 305, 51, 82, 287, 11, 315, 35, 220, 195, 311, 185, 5, 382, 73, 164, 14, 123, 205, 319, 383, 141, 83, 293, 43, 126, 225, 201, 358, 241, 86, 133, 249, 23, 292, 153, 52, 316, 90, 245, 122, 239, 248, 353, 165, 118, 105, 112, 75, 328, 281, 331, 103, 304, 148, 131, 102, 313, 215, 150, 53, 223, 143, 302, 155, 338, 308, 357, 238, 138, 183, 142, 15, 264, 177, 295, 327, 192, 270, 209, 378, 10, 255, 298, 65, 330, 258, 174, 354, 115, 79, 253, 377, 20, 289, 226, 110, 175, 135, 48, 279, 67, 351, 190, 219, 95, 221, 1, 231, 162, 54, 107, 163, 97, 260, 158, 337, 360, 345, 366, 113, 100, 300, 19, 12, 84, 182, 283, 50, 191, 40, 101, 365, 179, 250, 335, 346, 336, 106, 161, 277, 381, 30, 198, 291, 132, 72, 170, 203, 261, 59, 78, 111, 233, 136, 32, 96, 307, 204, 344, 134, 99, 139, 46, 368, 364, 42, 380, 39, 187, 85, 147, 240, 94, 169, 263, 80, 91, 224, 356, 98, 321, 196, 0, 212, 294, 227, 61, 62, 343, 208, 60, 234, 167, 252, 120, 269, 156, 57, 41, 379, 271, 237, 222, 17, 301, 348, 128, 7, 243, 369, 202, 34, 299, 318, 166, 3, 244, 323, 274, 24, 27, 2, 56, 213, 36, 272, 361, 254, 325, 45, 363, 114, 268, 33, 49, 31, 116, 89, 332, 235] + 24: [154, 5, 62, 135, 91, 142, 73, 310, 63, 295, 280, 108, 333, 175, 151, 171, 193, 236, 141, 243, 364, 81, 115, 166, 265, 112, 186, 354, 144, 138, 49, 114, 297, 197, 51, 181, 72, 10, 352, 262, 288, 256, 90, 199, 17, 374, 379, 383, 252, 28, 255, 159, 54, 350, 130, 206, 140, 168, 312, 218, 365, 212, 161, 329, 291, 367, 119, 200, 40, 21, 198, 241, 9, 259, 324, 231, 202, 273, 274, 380, 245, 147, 269, 239, 134, 276, 253, 99, 57, 248, 43, 143, 349, 82, 127, 264, 246, 328, 286, 375, 192, 15, 355, 177, 307, 169, 313, 244, 215, 275, 196, 136, 331, 25, 80, 228, 371, 100, 129, 298, 20, 95, 361, 230, 31, 306, 356, 113, 272, 321, 13, 66, 103, 282, 296, 281, 148, 183, 125, 240, 33, 214, 133, 314, 7, 110, 381, 301, 26, 366, 254, 224, 345, 216, 153, 339, 258, 170, 363, 373, 67, 109, 180, 165, 227, 152, 158, 353, 223, 211, 360, 334, 343, 344, 318, 377, 182, 58, 235, 304, 242, 65, 23, 303, 8, 42, 101, 201, 250, 69, 261, 179, 0, 87, 157, 220, 137, 195, 93, 46, 149, 287, 376, 16, 268, 309, 203, 173, 188, 50, 209, 229, 300, 117, 305, 232, 52, 97, 139, 294, 283, 204, 61, 131, 338, 12, 83, 222, 315, 270, 4, 14, 284, 102, 217, 187, 98, 285, 219, 251, 92, 290, 185, 104, 319, 53, 299, 226, 96, 205, 221, 323, 302, 346, 267, 86, 32, 191, 132, 362, 167, 289, 358, 327, 6, 128, 122, 266, 370, 24, 382, 156, 336, 77, 59, 278, 56, 194, 348, 88, 94, 19, 342, 247, 162, 316, 2, 34, 22, 189, 337, 64, 29, 164, 190, 308, 293, 145, 178, 84, 292, 37, 335, 107, 76, 160, 44, 18, 332, 208, 322, 368, 372, 85, 210, 121, 237, 213, 325, 41, 39, 48, 279, 233, 1, 225, 79, 176, 126, 163, 238, 68, 75, 340, 45, 36, 263, 369, 172, 27, 146, 277, 35, 260, 351, 3, 326, 311, 11, 30, 38, 120, 249, 359, 317, 184, 124, 347, 89, 155, 357, 70, 111, 105, 207, 78, 71, 106, 341, 118, 257, 116, 47, 60, 271, 150, 320, 55, 234, 378, 174, 123, 330, 74] + 25: [315, 331, 310, 159, 166, 117, 70, 371, 224, 363, 219, 151, 338, 19, 165, 239, 220, 267, 277, 123, 178, 91, 158, 326, 269, 253, 138, 196, 347, 187, 259, 257, 200, 251, 337, 167, 45, 41, 265, 169, 206, 195, 43, 382, 129, 233, 156, 263, 379, 216, 381, 373, 153, 321, 114, 278, 345, 202, 94, 79, 243, 111, 157, 208, 64, 264, 42, 266, 270, 303, 121, 189, 282, 246, 280, 314, 120, 234, 99, 126, 261, 203, 312, 308, 181, 31, 340, 292, 93, 311, 118, 286, 88, 288, 255, 44, 72, 78, 113, 190, 296, 368, 197, 256, 146, 307, 304, 81, 185, 201, 150, 361, 22, 281, 16, 128, 57, 335, 106, 170, 139, 334, 2, 171, 273, 318, 154, 84, 244, 161, 291, 237, 204, 328, 144, 300, 127, 35, 320, 175, 155, 3, 301, 194, 228, 298, 39, 10, 305, 230, 302, 377, 89, 97, 242, 295, 354, 365, 149, 53, 275, 75, 122, 40, 137, 48, 108, 47, 247, 268, 61, 112, 7, 59, 136, 341, 323, 218, 20, 319, 324, 134, 69, 238, 110, 177, 21, 46, 142, 95, 104, 148, 317, 168, 332, 77, 160, 380, 262, 346, 229, 367, 140, 191, 383, 213, 28, 212, 27, 186, 86, 210, 188, 63, 217, 50, 180, 258, 162, 376, 235, 32, 18, 306, 240, 60, 103, 115, 199, 290, 116, 370, 358, 353, 183, 96, 92, 205, 285, 182, 276, 83, 375, 152, 5, 102, 207, 222, 209, 164, 135, 132, 221, 15, 254, 24, 17, 271, 250, 226, 174, 344, 90, 272, 23, 65, 30, 25, 145, 33, 351, 294, 76, 54, 359, 336, 316, 297, 58, 67, 74, 109, 34, 364, 322, 1, 374, 13, 372, 98, 101, 327, 73, 313, 100, 131, 231, 279, 176, 352, 350, 37, 339, 105, 193, 248, 362, 87, 236, 147, 26, 241, 107, 62, 82, 0, 232, 8, 287, 215, 125, 366, 143, 211, 55, 330, 274, 356, 223, 51, 325, 293, 289, 360, 71, 329, 214, 192, 299, 309, 333, 38, 378, 9, 80, 227, 348, 369, 68, 357, 14, 355, 179, 260, 124, 283, 349, 119, 12, 198, 29, 284, 85, 52, 49, 66, 342, 252, 56, 343, 4, 225, 130, 184, 133, 11, 249, 163, 36, 173, 245, 141, 172, 6] + 26: [136, 204, 191, 174, 66, 318, 41, 129, 189, 368, 165, 335, 326, 258, 377, 95, 248, 54, 68, 86, 96, 106, 111, 139, 237, 224, 130, 135, 167, 359, 319, 306, 24, 57, 357, 115, 75, 280, 7, 140, 297, 39, 91, 282, 257, 132, 169, 170, 89, 342, 69, 266, 30, 332, 176, 166, 152, 194, 215, 267, 123, 232, 121, 255, 59, 77, 270, 42, 381, 168, 175, 209, 269, 271, 16, 38, 22, 200, 128, 133, 79, 118, 239, 107, 187, 301, 157, 105, 40, 154, 119, 283, 23, 379, 374, 375, 217, 349, 254, 137, 226, 382, 156, 26, 367, 173, 71, 161, 35, 196, 182, 300, 354, 74, 112, 199, 295, 366, 207, 218, 172, 281, 160, 183, 223, 126, 180, 178, 44, 241, 365, 113, 212, 231, 372, 220, 225, 72, 163, 205, 330, 84, 286, 228, 90, 233, 149, 316, 61, 146, 247, 360, 2, 186, 192, 277, 0, 50, 312, 197, 317, 334, 193, 274, 85, 222, 203, 234, 80, 5, 162, 33, 355, 92, 265, 179, 229, 122, 285, 10, 73, 256, 383, 49, 125, 347, 206, 314, 211, 345, 213, 242, 6, 320, 338, 344, 227, 245, 62, 48, 190, 327, 51, 104, 76, 93, 298, 219, 325, 8, 100, 101, 284, 310, 322, 246, 292, 150, 304, 276, 238, 376, 55, 148, 353, 103, 201, 195, 378, 114, 363, 78, 340, 43, 244, 216, 4, 28, 210, 261, 380, 230, 263, 279, 293, 36, 110, 198, 262, 97, 343, 159, 290, 350, 19, 264, 352, 64, 333, 296, 14, 46, 273, 305, 11, 339, 70, 337, 369, 108, 328, 272, 151, 356, 127, 65, 45, 370, 117, 329, 362, 346, 29, 13, 145, 147, 288, 348, 124, 253, 184, 94, 307, 88, 131, 259, 321, 102, 221, 81, 98, 32, 143, 3, 141, 9, 303, 324, 208, 164, 309, 351, 18, 291, 63, 116, 278, 158, 82, 138, 323, 99, 20, 15, 171, 144, 25, 302, 364, 34, 331, 358, 243, 188, 52, 260, 53, 181, 214, 120, 308, 249, 83, 311, 287, 27, 294, 87, 202, 37, 153, 12, 185, 275, 361, 373, 250, 235, 60, 299, 236, 21, 371, 289, 341, 315, 67, 31, 142, 251, 268, 17, 252, 177, 134, 336, 47, 109, 240, 56, 313, 58, 155, 1] + 27: [103, 290, 343, 322, 109, 261, 114, 296, 147, 9, 84, 139, 280, 302, 158, 345, 295, 92, 270, 240, 244, 262, 310, 338, 225, 58, 357, 188, 112, 323, 72, 82, 187, 370, 6, 274, 248, 121, 365, 35, 113, 186, 195, 192, 11, 169, 285, 342, 286, 117, 26, 181, 55, 21, 238, 89, 98, 155, 172, 327, 20, 318, 3, 105, 152, 74, 257, 64, 138, 363, 39, 53, 12, 145, 204, 24, 97, 242, 99, 78, 241, 88, 175, 127, 111, 66, 13, 130, 184, 264, 273, 166, 19, 371, 60, 137, 312, 104, 164, 366, 143, 272, 383, 308, 141, 15, 271, 5, 362, 249, 167, 79, 259, 293, 62, 283, 255, 57, 100, 146, 90, 301, 233, 358, 207, 48, 149, 83, 373, 161, 54, 215, 276, 44, 275, 306, 231, 253, 86, 334, 210, 298, 163, 176, 234, 81, 300, 376, 222, 230, 266, 115, 140, 42, 309, 174, 46, 2, 321, 153, 144, 359, 352, 94, 189, 221, 355, 220, 37, 354, 374, 128, 224, 251, 324, 156, 284, 133, 126, 304, 267, 171, 47, 361, 132, 319, 41, 372, 194, 148, 134, 180, 177, 34, 350, 49, 179, 75, 76, 294, 25, 27, 316, 136, 45, 299, 250, 123, 344, 346, 150, 16, 87, 160, 206, 200, 4, 211, 31, 119, 348, 228, 380, 258, 297, 356, 7, 291, 268, 43, 120, 107, 33, 281, 157, 142, 108, 170, 209, 203, 263, 341, 70, 315, 131, 336, 303, 77, 278, 317, 368, 326, 375, 106, 199, 205, 159, 185, 223, 382, 165, 311, 243, 212, 61, 227, 279, 320, 213, 183, 85, 232, 32, 191, 162, 379, 360, 239, 28, 367, 14, 96, 229, 68, 198, 277, 40, 235, 265, 80, 59, 269, 202, 237, 340, 216, 325, 337, 328, 246, 122, 18, 307, 217, 116, 154, 8, 330, 226, 314, 378, 236, 168, 349, 219, 135, 292, 190, 260, 196, 23, 193, 91, 329, 124, 67, 29, 95, 288, 208, 22, 110, 1, 331, 51, 247, 73, 30, 333, 71, 369, 335, 381, 339, 332, 305, 254, 214, 218, 347, 38, 0, 256, 282, 50, 289, 52, 56, 17, 63, 252, 69, 351, 197, 377, 182, 178, 102, 201, 118, 125, 10, 287, 129, 101, 353, 151, 313, 65, 364, 173, 245, 36, 93] + 28: [161, 289, 182, 247, 178, 256, 249, 381, 12, 297, 79, 351, 72, 76, 141, 313, 9, 138, 18, 60, 86, 115, 150, 156, 250, 253, 346, 59, 261, 285, 32, 216, 335, 222, 264, 274, 175, 170, 73, 266, 234, 232, 373, 53, 318, 336, 338, 343, 26, 248, 97, 108, 325, 228, 157, 125, 349, 316, 14, 265, 69, 148, 342, 372, 96, 251, 29, 101, 202, 11, 147, 180, 185, 0, 226, 307, 236, 71, 129, 337, 56, 43, 282, 327, 44, 362, 332, 151, 287, 17, 85, 45, 360, 40, 173, 207, 168, 312, 152, 113, 128, 227, 5, 90, 220, 273, 380, 194, 46, 82, 143, 211, 154, 323, 292, 55, 174, 350, 54, 190, 344, 355, 203, 214, 50, 42, 158, 361, 370, 193, 315, 290, 208, 81, 21, 363, 31, 218, 293, 252, 267, 65, 319, 353, 92, 4, 188, 176, 37, 320, 165, 130, 41, 367, 268, 68, 260, 98, 322, 334, 294, 348, 303, 281, 275, 189, 87, 215, 356, 321, 22, 93, 357, 3, 100, 167, 105, 13, 341, 184, 112, 64, 36, 210, 136, 149, 8, 146, 94, 83, 124, 257, 153, 197, 283, 155, 369, 219, 352, 67, 299, 19, 48, 121, 49, 57, 61, 270, 382, 119, 38, 229, 102, 145, 164, 279, 70, 241, 328, 120, 224, 223, 204, 235, 209, 217, 300, 1, 142, 305, 296, 15, 34, 80, 131, 276, 126, 181, 198, 311, 317, 286, 324, 103, 7, 368, 51, 88, 271, 20, 278, 118, 179, 295, 74, 104, 377, 330, 225, 237, 277, 187, 262, 329, 255, 99, 196, 122, 374, 137, 58, 160, 326, 376, 206, 333, 171, 358, 177, 133, 144, 364, 284, 347, 291, 309, 62, 298, 306, 25, 77, 288, 272, 95, 200, 135, 242, 238, 127, 27, 183, 375, 169, 230, 109, 117, 39, 47, 123, 254, 186, 359, 302, 52, 201, 195, 75, 166, 63, 28, 240, 308, 16, 78, 6, 23, 205, 280, 331, 89, 110, 192, 140, 354, 212, 379, 162, 107, 239, 172, 263, 378, 383, 163, 221, 340, 134, 91, 365, 159, 106, 84, 366, 345, 246, 139, 35, 304, 301, 10, 243, 30, 310, 233, 199, 191, 116, 339, 269, 33, 114, 244, 245, 314, 258, 66, 24, 213, 231, 259, 111, 371, 2, 132] + 29: [118, 124, 262, 326, 188, 123, 50, 139, 131, 78, 3, 242, 348, 114, 327, 62, 270, 290, 301, 100, 107, 241, 66, 91, 244, 283, 186, 112, 332, 38, 350, 232, 329, 104, 292, 31, 36, 368, 56, 69, 307, 30, 92, 330, 165, 281, 152, 176, 361, 383, 333, 160, 275, 207, 4, 273, 136, 345, 125, 344, 28, 135, 250, 13, 356, 246, 109, 80, 231, 206, 355, 285, 197, 89, 354, 16, 159, 370, 86, 378, 200, 59, 267, 269, 77, 46, 191, 338, 199, 113, 251, 41, 279, 268, 247, 108, 73, 255, 258, 148, 48, 216, 193, 298, 156, 272, 353, 142, 224, 315, 103, 42, 15, 162, 35, 223, 331, 342, 98, 249, 196, 11, 120, 351, 57, 237, 382, 61, 178, 349, 380, 32, 47, 52, 79, 84, 121, 168, 211, 358, 2, 335, 256, 6, 336, 127, 379, 208, 266, 163, 68, 324, 128, 347, 181, 313, 217, 369, 209, 9, 187, 376, 252, 133, 201, 64, 289, 26, 318, 311, 76, 320, 180, 190, 102, 132, 257, 18, 97, 317, 22, 93, 381, 158, 179, 149, 337, 189, 254, 1, 274, 177, 51, 23, 25, 202, 346, 63, 319, 248, 215, 300, 150, 328, 83, 157, 111, 7, 192, 219, 45, 164, 245, 372, 284, 185, 137, 173, 54, 110, 243, 87, 341, 360, 116, 225, 264, 340, 229, 126, 213, 288, 233, 306, 115, 234, 5, 260, 367, 322, 277, 282, 365, 74, 65, 343, 210, 94, 182, 280, 151, 205, 359, 302, 377, 161, 261, 194, 141, 325, 27, 291, 37, 212, 323, 44, 195, 129, 366, 175, 204, 314, 218, 146, 228, 371, 169, 312, 287, 70, 239, 308, 184, 49, 8, 221, 305, 240, 198, 236, 88, 339, 140, 134, 145, 259, 85, 72, 101, 20, 316, 40, 53, 153, 253, 17, 154, 96, 21, 321, 117, 363, 203, 60, 271, 309, 174, 375, 82, 172, 106, 222, 310, 105, 373, 10, 147, 166, 374, 334, 263, 155, 95, 81, 138, 29, 295, 67, 14, 33, 119, 235, 293, 278, 12, 171, 276, 167, 238, 230, 55, 220, 227, 0, 71, 296, 75, 226, 214, 39, 304, 362, 58, 265, 183, 24, 357, 364, 34, 19, 90, 294, 122, 170, 303, 299, 352, 43, 99, 297, 130, 286, 143, 144] + 30: [208, 160, 91, 142, 280, 73, 145, 342, 304, 287, 291, 255, 149, 229, 8, 21, 22, 31, 36, 40, 47, 88, 94, 101, 239, 274, 60, 167, 20, 111, 284, 318, 371, 381, 370, 173, 352, 328, 34, 131, 118, 123, 128, 134, 146, 185, 189, 218, 140, 59, 321, 283, 159, 336, 193, 344, 258, 329, 163, 109, 248, 166, 267, 57, 327, 178, 126, 235, 87, 176, 219, 237, 266, 242, 361, 147, 362, 138, 256, 320, 369, 96, 179, 310, 244, 288, 230, 32, 69, 368, 198, 265, 119, 201, 259, 268, 65, 76, 78, 234, 164, 340, 77, 58, 195, 67, 132, 335, 129, 332, 175, 377, 264, 85, 3, 16, 42, 191, 227, 269, 245, 333, 196, 188, 33, 97, 12, 71, 217, 281, 278, 114, 197, 115, 290, 202, 324, 323, 314, 300, 313, 251, 238, 322, 37, 213, 135, 11, 68, 216, 317, 275, 204, 356, 104, 383, 172, 55, 79, 276, 279, 125, 81, 190, 63, 252, 301, 334, 262, 184, 75, 206, 41, 303, 221, 192, 136, 285, 325, 103, 294, 48, 374, 95, 139, 155, 122, 308, 171, 4, 30, 305, 187, 124, 243, 2, 10, 17, 312, 152, 236, 260, 157, 84, 14, 66, 92, 110, 121, 116, 130, 105, 112, 56, 367, 380, 358, 254, 366, 295, 346, 250, 89, 177, 141, 226, 200, 353, 359, 286, 44, 360, 247, 90, 82, 355, 207, 199, 148, 345, 53, 158, 13, 168, 45, 316, 341, 357, 9, 296, 23, 261, 51, 293, 50, 25, 220, 161, 273, 15, 232, 372, 27, 70, 214, 133, 222, 46, 28, 271, 74, 100, 210, 382, 343, 108, 319, 349, 282, 154, 183, 83, 5, 169, 102, 270, 26, 307, 52, 289, 272, 378, 64, 212, 240, 253, 151, 277, 174, 38, 376, 375, 297, 311, 107, 326, 117, 127, 330, 39, 61, 331, 35, 62, 29, 257, 223, 302, 49, 6, 215, 18, 98, 54, 150, 241, 7, 194, 263, 348, 364, 153, 113, 0, 93, 354, 143, 156, 137, 24, 246, 315, 43, 1, 299, 292, 365, 72, 306, 211, 181, 337, 350, 203, 165, 233, 106, 338, 182, 363, 351, 373, 86, 225, 144, 205, 224, 339, 309, 170, 298, 231, 162, 180, 249, 186, 347, 99, 209, 19, 120, 80, 228, 379] + 31: [0, 186, 208, 258, 249, 40, 277, 202, 288, 8, 153, 305, 352, 226, 210, 327, 280, 35, 18, 104, 343, 132, 207, 329, 159, 364, 219, 119, 360, 112, 254, 55, 318, 307, 383, 12, 362, 228, 50, 101, 337, 181, 274, 319, 117, 67, 172, 350, 128, 292, 339, 140, 75, 214, 290, 353, 169, 46, 41, 142, 51, 108, 91, 238, 259, 342, 235, 282, 357, 154, 203, 229, 233, 265, 133, 150, 155, 286, 223, 138, 57, 205, 373, 336, 273, 11, 43, 308, 42, 64, 356, 114, 167, 200, 341, 347, 26, 331, 116, 268, 206, 72, 269, 244, 98, 65, 287, 303, 73, 177, 83, 375, 240, 113, 381, 184, 242, 13, 165, 120, 201, 125, 121, 270, 22, 283, 102, 189, 111, 243, 222, 256, 92, 195, 323, 275, 47, 309, 135, 109, 313, 27, 297, 180, 68, 320, 379, 267, 209, 34, 164, 236, 74, 368, 176, 374, 141, 183, 237, 220, 263, 367, 99, 52, 255, 30, 377, 246, 100, 248, 279, 95, 349, 188, 299, 211, 63, 2, 322, 213, 123, 110, 376, 48, 224, 157, 315, 79, 382, 311, 199, 37, 166, 227, 241, 25, 96, 61, 56, 78, 23, 245, 15, 284, 16, 285, 6, 371, 80, 139, 321, 127, 295, 332, 230, 90, 324, 76, 192, 215, 39, 170, 338, 82, 231, 366, 105, 312, 217, 354, 14, 130, 281, 178, 340, 193, 44, 359, 358, 212, 158, 87, 19, 250, 146, 60, 348, 106, 93, 81, 122, 118, 84, 29, 272, 1, 107, 251, 9, 7, 344, 161, 86, 289, 252, 185, 36, 326, 325, 58, 334, 145, 38, 66, 77, 97, 369, 361, 59, 194, 198, 129, 24, 53, 372, 218, 163, 294, 143, 261, 221, 187, 330, 378, 131, 33, 136, 175, 137, 314, 302, 196, 271, 264, 304, 160, 168, 293, 126, 174, 5, 89, 115, 70, 32, 147, 328, 171, 355, 28, 232, 134, 182, 21, 45, 260, 247, 204, 103, 298, 262, 156, 197, 257, 94, 149, 239, 363, 3, 335, 173, 333, 276, 296, 152, 88, 31, 316, 291, 49, 266, 346, 306, 345, 234, 191, 124, 179, 10, 148, 278, 144, 365, 69, 317, 253, 62, 380, 17, 54, 190, 225, 310, 4, 20, 351, 162, 85, 300, 216, 370, 301, 151, 71] + 32: [63, 256, 371, 6, 56, 339, 43, 128, 119, 170, 76, 98, 328, 162, 167, 169, 177, 181, 244, 245, 251, 262, 265, 296, 367, 65, 126, 273, 301, 325, 175, 150, 351, 84, 366, 154, 282, 195, 365, 0, 52, 180, 217, 15, 326, 173, 312, 373, 352, 229, 2, 380, 140, 335, 345, 303, 50, 178, 286, 3, 381, 172, 361, 197, 13, 330, 220, 364, 358, 295, 383, 375, 94, 135, 271, 219, 89, 97, 232, 168, 85, 206, 250, 336, 70, 309, 236, 221, 125, 248, 35, 278, 23, 17, 290, 379, 218, 234, 310, 274, 348, 151, 109, 7, 36, 101, 45, 32, 80, 10, 317, 9, 191, 222, 67, 79, 216, 47, 338, 57, 148, 246, 179, 257, 319, 279, 308, 189, 227, 25, 239, 127, 302, 22, 372, 99, 299, 223, 196, 138, 259, 112, 93, 300, 33, 155, 323, 12, 186, 211, 228, 313, 55, 26, 48, 183, 115, 340, 363, 42, 287, 341, 160, 343, 46, 74, 194, 316, 92, 288, 146, 8, 141, 283, 233, 306, 72, 231, 324, 382, 81, 4, 77, 82, 103, 304, 355, 149, 260, 192, 11, 333, 285, 275, 362, 121, 16, 202, 131, 113, 266, 104, 281, 182, 350, 24, 37, 311, 102, 356, 261, 334, 1, 368, 27, 54, 347, 322, 187, 66, 359, 96, 156, 294, 254, 69, 314, 59, 240, 337, 349, 136, 243, 117, 201, 208, 277, 88, 226, 90, 190, 329, 106, 280, 342, 184, 143, 289, 212, 354, 157, 86, 249, 129, 62, 5, 132, 20, 374, 224, 31, 95, 235, 114, 241, 166, 209, 360, 204, 225, 78, 19, 369, 71, 64, 237, 39, 205, 193, 210, 152, 238, 305, 188, 60, 18, 159, 14, 163, 344, 108, 130, 214, 174, 87, 120, 297, 199, 332, 230, 353, 276, 139, 320, 124, 376, 44, 203, 321, 272, 267, 145, 51, 107, 49, 21, 41, 83, 331, 357, 91, 111, 213, 293, 29, 247, 198, 110, 284, 73, 264, 118, 100, 161, 318, 123, 122, 164, 346, 370, 61, 263, 116, 252, 171, 185, 269, 142, 53, 315, 378, 144, 307, 153, 134, 255, 133, 215, 207, 68, 327, 38, 268, 200, 40, 75, 258, 58, 298, 292, 28, 253, 176, 30, 158, 34, 165, 105, 270, 377, 137, 242, 291, 147] + 33: [38, 68, 226, 376, 151, 222, 144, 230, 50, 370, 43, 28, 101, 318, 300, 119, 365, 274, 359, 223, 265, 293, 333, 372, 91, 153, 342, 201, 335, 160, 192, 9, 138, 216, 156, 279, 58, 215, 53, 295, 207, 168, 341, 195, 44, 47, 95, 220, 339, 176, 323, 84, 127, 17, 40, 217, 355, 383, 382, 180, 137, 66, 271, 327, 243, 381, 242, 315, 296, 203, 102, 277, 36, 51, 42, 179, 352, 273, 210, 289, 354, 109, 80, 367, 60, 292, 52, 67, 316, 245, 312, 253, 374, 0, 239, 287, 319, 328, 31, 73, 164, 233, 150, 268, 290, 16, 124, 209, 340, 106, 366, 248, 20, 132, 76, 59, 337, 304, 302, 34, 260, 167, 377, 82, 348, 37, 1, 356, 311, 173, 254, 104, 373, 345, 349, 278, 136, 23, 224, 118, 275, 99, 77, 129, 177, 86, 191, 107, 147, 361, 25, 237, 235, 93, 45, 375, 13, 234, 371, 70, 10, 238, 332, 98, 214, 309, 100, 264, 362, 89, 145, 379, 108, 48, 33, 208, 8, 187, 317, 55, 200, 249, 56, 103, 135, 346, 310, 62, 306, 321, 185, 83, 64, 94, 63, 39, 314, 5, 30, 193, 232, 351, 276, 262, 152, 213, 71, 166, 194, 74, 225, 175, 363, 2, 284, 69, 380, 267, 280, 96, 241, 270, 218, 155, 116, 202, 347, 162, 88, 170, 24, 285, 128, 298, 120, 257, 171, 7, 263, 236, 229, 307, 331, 297, 358, 255, 35, 131, 288, 90, 4, 157, 360, 15, 336, 112, 221, 111, 27, 178, 165, 344, 357, 161, 148, 154, 259, 183, 125, 169, 204, 334, 305, 329, 198, 256, 186, 368, 113, 110, 41, 130, 184, 244, 199, 57, 294, 142, 81, 299, 320, 325, 139, 266, 338, 227, 18, 211, 22, 196, 212, 364, 3, 65, 6, 182, 291, 353, 378, 172, 46, 32, 343, 283, 75, 231, 146, 190, 141, 133, 14, 205, 123, 19, 114, 122, 350, 324, 219, 286, 78, 134, 250, 228, 282, 12, 115, 301, 140, 121, 322, 21, 54, 308, 126, 303, 79, 246, 369, 105, 330, 26, 252, 181, 188, 247, 92, 313, 272, 159, 251, 258, 240, 158, 72, 206, 87, 117, 61, 149, 174, 197, 163, 49, 281, 29, 261, 11, 85, 326, 97, 269, 143, 189] + 34: [108, 194, 97, 300, 134, 12, 166, 195, 48, 247, 180, 10, 383, 314, 265, 93, 176, 104, 71, 252, 129, 14, 76, 119, 179, 223, 178, 2, 267, 124, 174, 197, 336, 72, 367, 352, 62, 186, 275, 86, 118, 205, 141, 54, 163, 273, 122, 132, 16, 354, 148, 284, 238, 77, 210, 136, 280, 328, 204, 361, 42, 298, 68, 219, 319, 165, 171, 128, 216, 343, 160, 189, 249, 151, 145, 138, 79, 6, 125, 184, 326, 133, 236, 333, 308, 127, 358, 88, 289, 380, 52, 55, 120, 106, 376, 193, 306, 237, 304, 40, 295, 8, 65, 81, 183, 143, 346, 364, 232, 19, 221, 85, 228, 100, 242, 112, 1, 325, 377, 202, 15, 47, 158, 362, 315, 78, 162, 329, 327, 137, 227, 89, 5, 114, 23, 51, 233, 307, 278, 20, 212, 344, 177, 198, 340, 207, 239, 243, 220, 191, 201, 110, 24, 37, 231, 116, 50, 157, 46, 147, 269, 331, 370, 90, 257, 182, 235, 229, 366, 29, 311, 240, 246, 41, 87, 107, 84, 60, 150, 102, 263, 277, 105, 264, 282, 75, 57, 155, 156, 303, 7, 211, 337, 382, 135, 291, 99, 335, 208, 144, 256, 317, 245, 214, 154, 164, 272, 192, 305, 255, 117, 297, 292, 94, 320, 213, 69, 372, 74, 167, 115, 271, 95, 290, 345, 172, 270, 169, 230, 294, 142, 266, 36, 226, 173, 342, 96, 188, 309, 363, 259, 61, 312, 203, 146, 92, 225, 373, 368, 251, 241, 0, 80, 63, 371, 268, 111, 274, 250, 253, 351, 234, 350, 224, 299, 190, 82, 293, 347, 222, 21, 58, 313, 33, 67, 287, 348, 322, 56, 98, 161, 286, 260, 339, 355, 276, 369, 279, 126, 196, 332, 185, 59, 288, 4, 53, 175, 217, 181, 209, 103, 140, 261, 152, 262, 353, 17, 285, 130, 338, 375, 365, 330, 44, 18, 31, 258, 11, 244, 356, 159, 206, 30, 359, 215, 318, 139, 296, 310, 34, 131, 9, 22, 349, 32, 316, 334, 3, 357, 321, 379, 254, 91, 153, 123, 199, 27, 170, 301, 323, 149, 64, 13, 28, 109, 39, 43, 378, 66, 341, 248, 218, 324, 25, 360, 83, 168, 200, 302, 187, 70, 73, 35, 113, 283, 45, 374, 101, 49, 26, 281, 38, 121, 381] + 35: [319, 211, 129, 186, 164, 159, 267, 36, 372, 122, 92, 214, 24, 301, 374, 87, 278, 89, 238, 33, 226, 199, 293, 177, 45, 62, 141, 346, 311, 331, 373, 272, 268, 313, 96, 277, 347, 376, 175, 35, 207, 115, 150, 148, 219, 2, 100, 343, 1, 80, 254, 153, 147, 335, 329, 360, 158, 30, 300, 350, 348, 166, 142, 338, 63, 245, 49, 107, 233, 280, 64, 228, 345, 309, 379, 21, 224, 326, 136, 160, 308, 294, 201, 261, 110, 383, 127, 84, 365, 149, 23, 10, 327, 252, 124, 85, 363, 48, 303, 4, 78, 307, 126, 187, 230, 213, 152, 287, 305, 330, 380, 25, 105, 316, 121, 77, 333, 247, 119, 291, 75, 340, 58, 324, 239, 234, 54, 321, 9, 215, 95, 172, 202, 216, 19, 88, 292, 370, 155, 288, 223, 0, 173, 359, 169, 281, 130, 138, 355, 194, 222, 357, 132, 190, 337, 235, 180, 34, 70, 344, 332, 377, 195, 165, 69, 188, 120, 304, 28, 113, 144, 47, 206, 349, 52, 102, 18, 328, 320, 170, 227, 204, 269, 43, 103, 253, 232, 183, 178, 208, 71, 13, 205, 322, 237, 257, 154, 265, 266, 134, 81, 104, 143, 323, 145, 382, 179, 73, 79, 248, 250, 255, 351, 242, 76, 381, 299, 72, 240, 55, 116, 364, 361, 37, 128, 61, 209, 11, 57, 106, 184, 279, 176, 135, 181, 263, 83, 371, 59, 283, 218, 276, 182, 286, 221, 375, 162, 27, 325, 171, 243, 16, 74, 369, 185, 285, 198, 189, 249, 358, 270, 123, 229, 284, 131, 203, 44, 368, 31, 352, 14, 163, 296, 5, 137, 32, 366, 264, 7, 196, 274, 259, 306, 97, 197, 258, 212, 200, 297, 342, 220, 318, 289, 46, 50, 98, 168, 192, 42, 90, 111, 91, 99, 17, 378, 29, 112, 167, 271, 336, 251, 60, 367, 86, 353, 56, 3, 41, 191, 312, 244, 298, 108, 256, 8, 82, 273, 217, 317, 161, 94, 262, 225, 51, 341, 40, 295, 53, 246, 67, 38, 114, 362, 157, 140, 282, 125, 12, 20, 117, 290, 315, 65, 151, 26, 354, 310, 210, 39, 146, 68, 133, 260, 314, 231, 15, 241, 275, 356, 93, 334, 302, 109, 339, 174, 66, 6, 156, 22, 139, 101, 193, 236, 118] + 36: [232, 380, 79, 42, 112, 49, 68, 171, 41, 211, 81, 335, 29, 58, 315, 249, 115, 8, 89, 181, 64, 170, 273, 302, 22, 59, 241, 97, 47, 33, 231, 167, 172, 153, 228, 217, 212, 17, 208, 239, 381, 337, 312, 87, 141, 222, 127, 360, 184, 370, 98, 207, 5, 262, 67, 138, 50, 168, 147, 305, 143, 157, 283, 281, 383, 357, 350, 126, 269, 306, 313, 375, 371, 197, 299, 56, 133, 261, 108, 293, 284, 342, 117, 174, 180, 39, 165, 365, 44, 6, 35, 137, 34, 96, 367, 336, 368, 294, 101, 40, 308, 32, 334, 194, 282, 289, 20, 100, 310, 28, 244, 51, 146, 77, 248, 253, 86, 124, 317, 270, 272, 252, 328, 369, 150, 85, 331, 191, 139, 183, 276, 116, 202, 257, 88, 23, 190, 158, 60, 14, 154, 330, 114, 145, 53, 192, 2, 274, 152, 93, 240, 219, 297, 245, 280, 237, 323, 36, 148, 80, 204, 259, 215, 110, 303, 27, 214, 16, 340, 131, 62, 196, 275, 92, 333, 9, 63, 164, 216, 179, 187, 52, 188, 119, 247, 169, 130, 226, 151, 227, 43, 268, 341, 107, 185, 258, 291, 205, 201, 246, 55, 69, 155, 66, 364, 104, 351, 78, 209, 265, 256, 61, 235, 286, 377, 25, 144, 223, 74, 320, 352, 332, 45, 319, 103, 91, 346, 195, 102, 200, 267, 163, 329, 338, 316, 254, 378, 4, 106, 37, 57, 12, 250, 260, 99, 54, 309, 111, 361, 300, 15, 271, 229, 0, 129, 220, 70, 295, 347, 118, 1, 128, 160, 279, 304, 372, 84, 230, 90, 314, 321, 362, 349, 3, 339, 177, 46, 76, 379, 48, 363, 236, 251, 213, 7, 327, 113, 376, 238, 225, 382, 296, 301, 31, 156, 94, 132, 298, 354, 224, 277, 264, 149, 243, 24, 109, 21, 120, 285, 203, 221, 290, 266, 193, 176, 242, 374, 125, 65, 345, 13, 322, 72, 343, 30, 71, 263, 318, 83, 189, 307, 75, 278, 136, 234, 292, 10, 178, 348, 210, 73, 287, 175, 166, 122, 121, 366, 140, 255, 38, 206, 18, 135, 173, 199, 355, 26, 353, 182, 186, 325, 311, 359, 326, 373, 233, 162, 288, 105, 95, 142, 11, 134, 344, 358, 159, 356, 198, 19, 324, 161, 82, 218, 123] + 37: [218, 104, 296, 111, 100, 207, 313, 210, 110, 216, 198, 106, 155, 307, 232, 263, 226, 112, 132, 236, 239, 288, 352, 383, 40, 171, 9, 181, 18, 85, 27, 41, 103, 356, 262, 94, 280, 53, 170, 259, 146, 177, 89, 358, 31, 375, 8, 75, 268, 178, 44, 361, 253, 147, 131, 322, 21, 365, 161, 15, 69, 255, 33, 249, 152, 378, 142, 214, 11, 276, 189, 120, 294, 151, 190, 174, 129, 327, 153, 359, 228, 149, 304, 285, 55, 321, 24, 267, 124, 260, 308, 184, 197, 186, 23, 99, 148, 243, 205, 292, 156, 252, 233, 192, 208, 173, 338, 135, 126, 334, 102, 105, 10, 266, 283, 225, 261, 90, 282, 209, 293, 273, 52, 305, 86, 245, 328, 257, 229, 64, 217, 333, 136, 37, 97, 325, 17, 303, 339, 57, 202, 381, 346, 251, 109, 66, 306, 179, 223, 360, 139, 330, 336, 51, 203, 30, 3, 331, 312, 274, 298, 215, 115, 63, 237, 45, 377, 56, 301, 364, 125, 272, 77, 185, 318, 32, 157, 258, 6, 141, 376, 83, 244, 38, 351, 254, 286, 374, 130, 227, 2, 246, 379, 71, 314, 48, 382, 302, 14, 343, 315, 287, 323, 36, 329, 224, 347, 284, 172, 138, 291, 310, 231, 281, 367, 275, 58, 319, 213, 372, 371, 47, 165, 163, 150, 122, 168, 39, 201, 247, 98, 279, 116, 160, 175, 35, 166, 20, 54, 67, 158, 240, 311, 341, 95, 355, 65, 118, 145, 76, 144, 143, 29, 206, 380, 317, 187, 13, 332, 81, 248, 180, 220, 84, 222, 188, 348, 46, 342, 140, 182, 235, 164, 370, 1, 43, 92, 335, 80, 373, 154, 26, 169, 183, 324, 326, 12, 211, 200, 0, 242, 289, 193, 114, 7, 194, 133, 316, 176, 82, 108, 34, 269, 212, 4, 256, 350, 191, 357, 300, 219, 25, 366, 340, 309, 362, 234, 59, 88, 117, 221, 19, 196, 61, 264, 49, 121, 241, 42, 368, 78, 295, 107, 70, 369, 238, 79, 113, 123, 363, 72, 344, 353, 93, 277, 320, 74, 230, 278, 87, 128, 195, 354, 91, 60, 101, 62, 28, 159, 50, 349, 162, 265, 199, 290, 16, 22, 204, 345, 127, 270, 73, 134, 167, 337, 271, 5, 299, 137, 297, 96, 119, 250, 68] + 38: [301, 169, 331, 219, 85, 359, 184, 48, 246, 376, 305, 337, 273, 215, 13, 202, 350, 153, 140, 318, 46, 60, 185, 237, 6, 309, 381, 77, 286, 352, 72, 307, 165, 229, 260, 33, 2, 313, 61, 297, 164, 65, 303, 162, 346, 251, 325, 380, 266, 93, 292, 63, 0, 300, 225, 222, 193, 57, 216, 289, 76, 353, 23, 122, 294, 205, 240, 112, 22, 160, 52, 191, 203, 187, 123, 220, 244, 344, 338, 99, 73, 105, 264, 111, 31, 242, 152, 214, 11, 183, 91, 320, 218, 198, 101, 210, 158, 4, 94, 194, 92, 206, 199, 355, 69, 262, 256, 37, 148, 41, 235, 79, 195, 80, 374, 113, 326, 315, 83, 161, 378, 10, 118, 248, 217, 135, 319, 233, 324, 59, 45, 298, 342, 197, 181, 173, 172, 295, 186, 299, 18, 20, 370, 176, 221, 282, 66, 115, 19, 27, 257, 106, 75, 358, 349, 179, 151, 84, 15, 296, 261, 231, 247, 254, 335, 365, 110, 339, 269, 62, 223, 8, 379, 166, 97, 329, 239, 21, 211, 142, 276, 356, 103, 302, 382, 25, 35, 343, 232, 132, 133, 121, 280, 89, 50, 274, 293, 212, 54, 90, 67, 29, 58, 308, 141, 28, 272, 200, 371, 311, 168, 363, 175, 291, 243, 304, 124, 333, 345, 263, 39, 17, 108, 327, 271, 95, 144, 7, 143, 180, 174, 81, 104, 375, 102, 3, 259, 116, 196, 340, 201, 154, 245, 332, 130, 145, 234, 288, 227, 208, 323, 136, 5, 241, 40, 334, 347, 82, 55, 236, 163, 207, 249, 312, 284, 277, 178, 322, 32, 328, 53, 361, 107, 224, 88, 156, 228, 306, 78, 360, 120, 100, 290, 357, 314, 209, 330, 64, 372, 114, 348, 86, 171, 278, 267, 125, 275, 150, 87, 367, 265, 170, 131, 238, 43, 252, 71, 12, 157, 377, 336, 38, 51, 383, 369, 226, 177, 279, 268, 341, 146, 321, 255, 368, 117, 190, 139, 182, 74, 44, 283, 70, 24, 167, 127, 98, 30, 126, 287, 285, 49, 317, 119, 16, 147, 362, 68, 96, 109, 26, 258, 250, 129, 47, 351, 192, 364, 281, 155, 159, 366, 189, 213, 354, 230, 373, 253, 137, 14, 9, 36, 34, 188, 134, 149, 310, 56, 204, 138, 316, 42, 128, 1, 270] + 39: [282, 267, 296, 250, 49, 202, 232, 138, 358, 364, 373, 88, 164, 377, 192, 303, 175, 123, 125, 308, 184, 89, 257, 44, 310, 156, 2, 362, 208, 59, 274, 340, 126, 371, 161, 61, 320, 363, 183, 27, 346, 211, 37, 67, 32, 90, 304, 209, 247, 348, 242, 329, 121, 77, 273, 140, 263, 353, 5, 33, 95, 4, 240, 91, 287, 78, 0, 219, 294, 21, 143, 361, 66, 82, 325, 145, 40, 163, 87, 127, 122, 69, 150, 301, 72, 97, 16, 181, 343, 55, 108, 284, 147, 237, 261, 149, 104, 234, 360, 86, 71, 76, 23, 191, 54, 152, 144, 213, 311, 275, 30, 214, 245, 159, 142, 64, 158, 124, 148, 52, 372, 280, 25, 272, 333, 369, 81, 248, 299, 367, 317, 63, 106, 133, 79, 265, 58, 111, 266, 233, 255, 332, 11, 165, 236, 96, 153, 271, 26, 300, 298, 383, 225, 74, 135, 162, 102, 173, 243, 60, 171, 117, 18, 252, 256, 57, 356, 22, 56, 99, 1, 155, 297, 141, 51, 84, 239, 137, 188, 226, 355, 193, 254, 120, 216, 107, 305, 359, 351, 262, 115, 9, 315, 176, 368, 119, 270, 17, 365, 375, 48, 101, 50, 307, 182, 203, 195, 327, 335, 168, 347, 238, 339, 269, 154, 198, 258, 100, 223, 279, 80, 98, 3, 146, 259, 112, 190, 281, 249, 212, 285, 309, 169, 109, 222, 20, 92, 349, 318, 321, 286, 43, 116, 13, 382, 324, 178, 7, 42, 251, 378, 322, 179, 94, 354, 130, 207, 196, 19, 83, 253, 350, 12, 186, 65, 85, 35, 151, 174, 172, 15, 381, 8, 323, 131, 62, 134, 167, 357, 220, 306, 288, 41, 206, 39, 210, 316, 201, 312, 379, 180, 215, 328, 295, 341, 235, 118, 336, 331, 366, 75, 228, 128, 221, 166, 283, 337, 246, 229, 326, 73, 344, 38, 313, 110, 139, 187, 374, 24, 290, 352, 28, 260, 177, 302, 289, 241, 68, 114, 217, 189, 160, 197, 370, 105, 380, 93, 314, 264, 218, 292, 277, 319, 200, 170, 31, 47, 276, 14, 293, 230, 338, 376, 291, 46, 157, 342, 129, 224, 29, 132, 36, 268, 113, 244, 204, 330, 227, 53, 278, 34, 199, 345, 45, 185, 6, 194, 70, 103, 231, 10, 136, 205, 334] + 40: [31, 316, 244, 155, 206, 37, 345, 246, 125, 135, 242, 285, 4, 298, 353, 170, 343, 105, 123, 149, 194, 214, 248, 300, 319, 213, 26, 158, 228, 362, 274, 289, 382, 56, 263, 151, 324, 237, 314, 78, 121, 266, 34, 134, 39, 24, 240, 376, 15, 181, 107, 88, 59, 363, 162, 322, 196, 309, 364, 25, 371, 281, 73, 6, 84, 268, 163, 22, 267, 191, 42, 92, 68, 260, 62, 161, 208, 91, 129, 277, 334, 302, 53, 299, 192, 305, 350, 261, 115, 184, 117, 106, 234, 223, 83, 249, 175, 280, 101, 156, 251, 17, 143, 63, 30, 308, 137, 57, 377, 356, 282, 85, 238, 276, 310, 77, 111, 313, 220, 72, 378, 340, 98, 279, 74, 28, 89, 80, 11, 199, 46, 82, 94, 287, 48, 47, 36, 256, 49, 40, 114, 176, 86, 312, 317, 357, 367, 202, 254, 3, 9, 168, 145, 200, 227, 189, 103, 327, 171, 60, 291, 288, 50, 113, 352, 381, 112, 331, 207, 379, 75, 215, 372, 58, 347, 341, 61, 148, 187, 365, 355, 339, 195, 301, 375, 361, 383, 257, 231, 96, 172, 225, 183, 132, 373, 297, 8, 100, 296, 10, 54, 216, 259, 150, 2, 29, 304, 18, 76, 348, 13, 66, 173, 315, 272, 335, 250, 226, 204, 52, 64, 224, 275, 326, 27, 236, 229, 43, 7, 120, 273, 293, 241, 0, 102, 35, 217, 380, 320, 146, 349, 33, 67, 337, 369, 258, 1, 336, 278, 38, 269, 138, 165, 338, 19, 131, 201, 23, 124, 218, 104, 152, 20, 126, 360, 346, 359, 265, 81, 235, 210, 271, 211, 205, 307, 95, 290, 333, 329, 139, 118, 14, 140, 203, 180, 87, 245, 344, 142, 179, 55, 283, 332, 198, 159, 209, 185, 370, 44, 41, 110, 93, 323, 222, 147, 141, 368, 230, 325, 109, 252, 5, 328, 108, 70, 247, 164, 97, 303, 136, 193, 12, 262, 65, 130, 318, 330, 233, 270, 212, 374, 295, 119, 342, 127, 197, 71, 21, 286, 160, 153, 351, 284, 219, 166, 188, 239, 178, 294, 264, 243, 51, 116, 186, 16, 174, 154, 45, 255, 144, 358, 190, 169, 128, 232, 90, 311, 321, 253, 221, 79, 99, 306, 167, 177, 366, 122, 133, 32, 182, 354, 157, 292, 69] + 41: [49, 11, 365, 172, 95, 326, 25, 174, 56, 322, 224, 107, 18, 317, 188, 292, 180, 28, 209, 157, 189, 118, 263, 357, 378, 196, 273, 92, 279, 306, 222, 336, 54, 235, 112, 268, 43, 128, 232, 77, 248, 98, 265, 145, 319, 135, 22, 309, 199, 87, 150, 76, 360, 253, 53, 243, 102, 55, 115, 328, 290, 5, 371, 272, 33, 165, 182, 329, 201, 346, 16, 40, 170, 185, 255, 285, 21, 198, 4, 126, 116, 254, 117, 266, 295, 246, 221, 41, 381, 48, 149, 375, 240, 88, 203, 178, 60, 305, 249, 124, 24, 108, 351, 353, 64, 256, 167, 288, 186, 19, 173, 154, 352, 332, 147, 14, 267, 361, 58, 247, 350, 122, 212, 97, 314, 337, 3, 23, 301, 276, 289, 380, 338, 81, 244, 300, 1, 141, 137, 29, 331, 362, 356, 296, 374, 205, 90, 293, 69, 44, 35, 194, 345, 93, 325, 8, 181, 6, 61, 359, 0, 72, 148, 370, 110, 162, 164, 31, 321, 215, 307, 223, 111, 206, 104, 202, 349, 241, 237, 144, 51, 160, 192, 303, 287, 68, 158, 67, 121, 312, 230, 120, 269, 119, 27, 50, 364, 311, 80, 42, 32, 282, 294, 250, 131, 261, 146, 15, 39, 259, 175, 373, 323, 211, 333, 152, 298, 159, 251, 327, 358, 347, 318, 36, 217, 379, 156, 138, 299, 94, 74, 99, 78, 297, 195, 17, 20, 377, 281, 216, 283, 275, 376, 368, 79, 262, 45, 219, 26, 225, 83, 133, 63, 277, 354, 37, 100, 320, 155, 238, 343, 270, 228, 233, 114, 129, 71, 200, 34, 12, 9, 239, 134, 70, 340, 330, 363, 130, 89, 214, 179, 13, 113, 171, 47, 106, 382, 132, 142, 366, 166, 168, 187, 334, 127, 140, 308, 125, 271, 242, 176, 208, 220, 123, 191, 207, 310, 231, 62, 341, 169, 52, 355, 197, 103, 84, 7, 280, 348, 264, 218, 226, 258, 344, 151, 91, 213, 342, 143, 234, 73, 204, 2, 177, 291, 278, 184, 369, 367, 57, 136, 302, 383, 286, 252, 46, 260, 210, 66, 153, 315, 75, 236, 38, 257, 245, 139, 372, 193, 227, 190, 30, 335, 86, 96, 161, 82, 324, 163, 313, 229, 339, 101, 284, 10, 59, 65, 105, 316, 274, 183, 109, 304, 85] + 42: [123, 137, 238, 310, 332, 364, 229, 104, 207, 115, 182, 24, 289, 27, 242, 294, 369, 376, 25, 71, 95, 254, 324, 339, 152, 247, 220, 342, 127, 274, 187, 147, 92, 160, 18, 311, 183, 196, 53, 343, 321, 75, 304, 249, 44, 382, 126, 121, 117, 211, 162, 263, 277, 30, 287, 201, 234, 118, 19, 64, 283, 5, 378, 293, 368, 186, 330, 11, 79, 0, 352, 193, 31, 295, 72, 374, 255, 373, 248, 17, 105, 322, 65, 340, 372, 315, 336, 355, 26, 145, 185, 367, 12, 83, 13, 46, 359, 20, 184, 179, 377, 319, 131, 140, 167, 181, 33, 192, 301, 89, 97, 204, 346, 284, 360, 136, 134, 110, 142, 40, 217, 375, 305, 41, 129, 21, 370, 135, 165, 335, 52, 297, 8, 323, 299, 251, 171, 318, 363, 288, 96, 70, 317, 252, 94, 154, 303, 150, 55, 327, 275, 259, 230, 308, 366, 380, 132, 246, 226, 291, 233, 206, 10, 334, 133, 57, 271, 273, 266, 210, 106, 264, 108, 189, 328, 224, 87, 36, 316, 381, 146, 9, 232, 16, 112, 290, 73, 198, 88, 100, 43, 15, 268, 188, 357, 63, 168, 208, 191, 354, 344, 326, 38, 39, 245, 219, 49, 258, 68, 56, 90, 285, 383, 35, 107, 86, 345, 2, 119, 279, 262, 203, 351, 320, 286, 356, 314, 227, 80, 231, 157, 58, 269, 84, 82, 128, 302, 353, 14, 91, 176, 37, 228, 151, 195, 59, 379, 221, 236, 216, 143, 76, 241, 62, 130, 313, 125, 194, 250, 292, 347, 190, 54, 240, 172, 138, 148, 282, 349, 48, 235, 178, 239, 338, 280, 166, 60, 307, 139, 205, 278, 175, 199, 296, 22, 153, 78, 365, 155, 333, 114, 215, 243, 209, 122, 141, 28, 161, 77, 149, 32, 180, 260, 7, 156, 261, 267, 124, 309, 276, 81, 66, 218, 158, 42, 212, 200, 281, 265, 4, 169, 103, 164, 257, 113, 371, 244, 298, 170, 120, 98, 3, 144, 47, 331, 74, 93, 256, 1, 350, 222, 6, 348, 341, 214, 173, 202, 223, 101, 225, 109, 34, 300, 213, 197, 23, 174, 253, 177, 163, 325, 111, 50, 116, 337, 306, 358, 29, 67, 329, 45, 362, 51, 237, 99, 61, 270, 272, 102, 312, 85, 69, 159, 361] + 43: [168, 332, 39, 57, 301, 338, 255, 282, 123, 149, 241, 47, 183, 125, 207, 122, 173, 355, 162, 106, 15, 104, 245, 348, 225, 344, 35, 94, 186, 318, 121, 73, 48, 299, 157, 145, 253, 101, 295, 8, 247, 211, 341, 172, 133, 176, 44, 354, 375, 27, 197, 366, 153, 227, 280, 32, 327, 271, 130, 107, 165, 167, 18, 76, 126, 109, 53, 199, 333, 212, 95, 3, 66, 369, 129, 100, 81, 203, 136, 85, 222, 140, 160, 237, 60, 266, 80, 170, 87, 132, 119, 69, 230, 58, 96, 214, 98, 164, 371, 252, 377, 111, 108, 272, 380, 204, 79, 28, 293, 143, 220, 329, 294, 233, 110, 357, 228, 362, 166, 195, 315, 23, 142, 239, 93, 29, 198, 150, 88, 286, 365, 334, 38, 210, 128, 116, 356, 268, 187, 179, 249, 383, 184, 169, 221, 351, 218, 216, 363, 342, 274, 368, 90, 277, 36, 243, 102, 56, 256, 288, 65, 19, 209, 248, 317, 139, 91, 192, 105, 43, 308, 349, 314, 264, 147, 89, 378, 135, 26, 67, 50, 217, 263, 292, 259, 287, 240, 258, 21, 236, 25, 206, 171, 298, 278, 59, 22, 323, 307, 374, 154, 306, 5, 63, 49, 70, 343, 370, 117, 163, 320, 276, 7, 270, 231, 361, 156, 297, 177, 37, 326, 322, 269, 250, 238, 86, 84, 155, 113, 45, 213, 11, 72, 358, 316, 260, 141, 281, 244, 182, 347, 30, 68, 112, 291, 223, 313, 2, 275, 339, 61, 10, 185, 251, 325, 226, 267, 381, 336, 285, 328, 340, 302, 376, 92, 75, 161, 20, 235, 127, 305, 77, 346, 219, 34, 345, 99, 120, 146, 159, 279, 118, 265, 51, 0, 42, 289, 331, 372, 190, 6, 254, 152, 138, 1, 4, 52, 234, 261, 273, 242, 46, 309, 31, 64, 134, 290, 312, 62, 208, 319, 373, 335, 283, 181, 97, 337, 55, 224, 191, 246, 83, 257, 196, 229, 180, 74, 304, 144, 158, 300, 311, 137, 303, 350, 382, 17, 14, 82, 13, 103, 178, 78, 321, 205, 200, 296, 12, 148, 284, 9, 215, 353, 193, 330, 114, 124, 352, 310, 54, 359, 324, 262, 364, 131, 360, 202, 379, 232, 115, 367, 194, 24, 16, 174, 175, 71, 151, 188, 201, 41, 40, 33, 189] + 44: [213, 257, 229, 86, 329, 337, 91, 296, 315, 380, 134, 126, 334, 309, 179, 357, 114, 375, 99, 330, 111, 308, 322, 374, 77, 25, 267, 311, 383, 142, 360, 138, 290, 70, 160, 381, 136, 40, 377, 237, 18, 7, 1, 194, 302, 31, 16, 116, 301, 258, 188, 135, 166, 141, 225, 207, 254, 100, 131, 221, 157, 185, 231, 156, 250, 51, 35, 373, 363, 356, 208, 332, 191, 17, 227, 140, 236, 249, 275, 89, 224, 201, 132, 261, 292, 74, 8, 3, 235, 232, 216, 115, 298, 293, 88, 303, 199, 0, 215, 331, 98, 182, 291, 117, 61, 55, 130, 120, 107, 90, 59, 169, 193, 367, 45, 27, 103, 127, 310, 12, 10, 167, 190, 328, 313, 118, 144, 94, 350, 219, 248, 269, 354, 108, 105, 211, 319, 4, 255, 251, 283, 259, 76, 112, 317, 64, 110, 279, 34, 306, 102, 14, 266, 39, 352, 359, 26, 15, 149, 341, 9, 53, 273, 119, 246, 123, 353, 209, 226, 97, 351, 368, 288, 294, 228, 340, 192, 150, 164, 177, 162, 165, 243, 20, 234, 137, 80, 153, 176, 152, 75, 339, 348, 238, 155, 29, 93, 38, 33, 121, 85, 173, 366, 369, 46, 370, 347, 23, 263, 183, 200, 233, 260, 335, 181, 13, 362, 168, 60, 277, 146, 242, 125, 355, 24, 349, 286, 54, 220, 278, 318, 32, 241, 62, 312, 151, 247, 139, 58, 161, 274, 346, 180, 252, 187, 289, 217, 265, 299, 287, 66, 174, 41, 92, 214, 104, 57, 204, 2, 239, 268, 361, 371, 189, 172, 170, 382, 68, 320, 264, 304, 158, 175, 163, 338, 96, 342, 282, 323, 81, 72, 256, 42, 195, 69, 37, 148, 106, 171, 145, 271, 43, 143, 63, 48, 276, 184, 364, 79, 87, 128, 83, 78, 336, 109, 314, 245, 5, 372, 73, 133, 129, 222, 21, 300, 82, 30, 333, 365, 324, 244, 122, 71, 67, 36, 147, 270, 218, 203, 321, 253, 22, 28, 297, 197, 206, 379, 284, 178, 376, 11, 212, 358, 343, 6, 154, 285, 202, 327, 196, 205, 305, 230, 49, 124, 50, 240, 47, 316, 295, 101, 223, 272, 84, 325, 280, 198, 95, 262, 186, 56, 378, 44, 344, 159, 19, 113, 345, 65, 326, 210, 307, 52, 281] + 45: [92, 378, 297, 39, 229, 308, 213, 106, 303, 58, 126, 282, 288, 314, 343, 225, 89, 270, 6, 325, 300, 174, 221, 347, 140, 233, 120, 47, 183, 228, 104, 372, 357, 109, 1, 19, 145, 38, 241, 289, 309, 238, 153, 305, 99, 66, 83, 100, 178, 318, 197, 85, 114, 351, 45, 338, 7, 44, 236, 340, 116, 358, 379, 271, 48, 377, 90, 97, 265, 75, 49, 72, 10, 301, 182, 103, 348, 331, 196, 88, 210, 157, 365, 257, 35, 205, 273, 169, 363, 113, 93, 186, 226, 148, 69, 17, 227, 173, 199, 146, 124, 248, 55, 74, 251, 86, 260, 222, 20, 170, 322, 374, 79, 268, 337, 280, 193, 216, 185, 138, 154, 191, 346, 160, 165, 258, 310, 30, 112, 108, 179, 376, 43, 46, 294, 127, 246, 190, 202, 82, 143, 366, 181, 14, 23, 324, 25, 369, 163, 275, 307, 277, 327, 135, 175, 2, 34, 291, 283, 381, 105, 364, 111, 62, 200, 77, 237, 192, 121, 383, 139, 262, 212, 234, 162, 245, 306, 80, 208, 123, 330, 319, 264, 231, 220, 371, 214, 102, 352, 172, 11, 40, 161, 81, 284, 171, 131, 224, 168, 16, 84, 281, 217, 22, 50, 252, 5, 342, 166, 78, 73, 56, 159, 278, 299, 64, 293, 95, 317, 176, 61, 128, 118, 51, 70, 267, 67, 96, 133, 296, 98, 255, 341, 147, 91, 31, 232, 13, 60, 117, 42, 164, 144, 247, 180, 209, 223, 122, 313, 242, 292, 63, 218, 87, 9, 263, 119, 312, 321, 125, 101, 287, 130, 355, 334, 329, 316, 356, 269, 52, 198, 115, 54, 156, 94, 302, 65, 304, 29, 151, 0, 368, 243, 382, 353, 259, 149, 244, 201, 276, 134, 375, 354, 107, 254, 155, 250, 345, 279, 24, 207, 59, 37, 211, 15, 203, 323, 132, 194, 335, 195, 137, 8, 373, 188, 339, 298, 272, 359, 36, 12, 41, 26, 4, 344, 290, 150, 57, 142, 3, 71, 152, 167, 53, 240, 336, 27, 367, 110, 141, 21, 349, 187, 326, 328, 249, 32, 253, 370, 256, 320, 177, 285, 362, 332, 189, 295, 28, 184, 266, 33, 350, 219, 274, 286, 380, 239, 158, 68, 360, 315, 261, 76, 230, 333, 129, 204, 215, 136, 18, 206, 311, 235, 361] + 46: [142, 170, 284, 38, 303, 102, 155, 260, 316, 50, 321, 268, 289, 313, 69, 190, 86, 163, 330, 134, 251, 225, 82, 122, 183, 210, 6, 254, 113, 350, 22, 372, 166, 13, 17, 193, 35, 222, 277, 364, 257, 337, 11, 72, 276, 70, 379, 356, 352, 188, 324, 68, 338, 39, 143, 121, 315, 128, 172, 57, 360, 255, 322, 145, 184, 15, 21, 126, 47, 349, 175, 213, 311, 152, 96, 344, 204, 345, 266, 117, 287, 192, 304, 103, 377, 160, 24, 348, 331, 262, 52, 75, 236, 54, 0, 5, 133, 335, 25, 209, 60, 55, 351, 131, 232, 137, 41, 198, 32, 308, 173, 317, 91, 370, 43, 358, 300, 123, 84, 27, 10, 140, 16, 104, 40, 165, 49, 280, 245, 156, 95, 272, 111, 97, 119, 71, 291, 244, 371, 31, 87, 116, 200, 36, 247, 66, 249, 191, 124, 76, 211, 108, 326, 369, 176, 157, 59, 92, 89, 56, 90, 93, 135, 229, 288, 179, 357, 88, 34, 319, 46, 202, 194, 199, 282, 130, 168, 180, 362, 216, 81, 329, 346, 293, 261, 278, 26, 281, 363, 248, 14, 147, 64, 106, 368, 246, 312, 203, 215, 195, 334, 259, 23, 292, 383, 78, 297, 9, 177, 309, 67, 94, 381, 185, 100, 302, 139, 354, 327, 239, 301, 146, 79, 33, 80, 4, 298, 235, 158, 263, 223, 20, 342, 339, 138, 77, 73, 306, 270, 220, 256, 367, 240, 51, 136, 44, 219, 376, 186, 109, 63, 237, 125, 144, 299, 159, 217, 361, 65, 127, 105, 212, 28, 231, 58, 230, 224, 274, 214, 374, 85, 8, 218, 197, 53, 273, 101, 373, 382, 74, 241, 98, 375, 341, 107, 279, 167, 252, 286, 149, 314, 238, 12, 242, 228, 62, 99, 42, 164, 267, 307, 141, 318, 153, 333, 171, 347, 366, 196, 296, 182, 325, 380, 207, 19, 221, 187, 18, 3, 120, 227, 37, 129, 340, 45, 328, 161, 265, 359, 234, 150, 258, 269, 29, 151, 61, 253, 343, 233, 332, 378, 114, 30, 115, 132, 295, 294, 250, 336, 290, 169, 7, 264, 305, 208, 275, 1, 174, 148, 323, 355, 110, 201, 154, 271, 2, 320, 83, 353, 118, 162, 205, 226, 206, 189, 243, 283, 285, 112, 310, 48, 181, 178, 365] + 47: [195, 198, 188, 159, 84, 124, 33, 12, 340, 117, 15, 303, 37, 310, 120, 308, 16, 30, 299, 8, 13, 191, 56, 288, 3, 279, 305, 274, 131, 122, 69, 223, 140, 322, 173, 183, 335, 60, 2, 10, 55, 304, 222, 82, 167, 214, 373, 326, 343, 375, 286, 362, 59, 114, 95, 166, 172, 0, 4, 23, 139, 281, 115, 200, 19, 275, 339, 317, 99, 283, 236, 209, 54, 35, 314, 344, 40, 251, 208, 89, 284, 184, 180, 233, 356, 273, 341, 329, 48, 189, 5, 277, 116, 232, 311, 369, 44, 192, 193, 215, 175, 327, 313, 377, 41, 199, 237, 164, 336, 225, 111, 20, 46, 57, 374, 361, 142, 221, 182, 246, 352, 143, 61, 316, 129, 342, 98, 42, 103, 255, 45, 106, 378, 276, 258, 260, 102, 72, 380, 372, 11, 125, 318, 293, 24, 179, 109, 353, 21, 229, 269, 104, 263, 364, 39, 354, 18, 259, 31, 186, 149, 216, 92, 38, 64, 165, 7, 62, 312, 187, 1, 368, 267, 363, 176, 67, 319, 50, 148, 53, 321, 163, 94, 248, 150, 266, 108, 178, 194, 206, 156, 146, 87, 347, 119, 14, 79, 346, 212, 26, 151, 370, 205, 52, 366, 121, 328, 185, 253, 152, 358, 86, 145, 217, 83, 360, 181, 271, 43, 345, 127, 110, 349, 330, 262, 285, 296, 51, 252, 70, 71, 49, 32, 242, 325, 174, 220, 244, 256, 113, 154, 75, 371, 309, 196, 118, 218, 351, 224, 323, 73, 355, 96, 243, 211, 107, 157, 160, 177, 207, 202, 264, 295, 128, 162, 137, 291, 333, 257, 287, 235, 226, 68, 298, 65, 25, 112, 80, 289, 278, 297, 365, 379, 290, 47, 247, 254, 381, 22, 134, 357, 227, 36, 238, 63, 265, 135, 249, 197, 338, 210, 348, 126, 332, 97, 334, 383, 350, 76, 88, 66, 261, 158, 300, 171, 34, 294, 376, 367, 155, 93, 324, 77, 101, 315, 91, 28, 292, 144, 190, 245, 74, 85, 270, 241, 272, 29, 359, 136, 280, 320, 168, 153, 230, 219, 228, 203, 138, 234, 141, 213, 306, 132, 9, 282, 58, 100, 268, 6, 169, 382, 307, 331, 170, 90, 240, 123, 250, 301, 231, 204, 17, 27, 201, 147, 337, 130, 81, 302, 133, 105, 78, 161, 239] + 48: [17, 26, 226, 83, 189, 254, 229, 322, 120, 329, 168, 22, 119, 84, 144, 382, 93, 3, 309, 338, 183, 220, 129, 227, 271, 249, 44, 133, 223, 186, 290, 179, 257, 170, 4, 339, 36, 198, 169, 108, 310, 163, 201, 225, 68, 126, 30, 308, 140, 297, 109, 295, 6, 264, 252, 161, 285, 255, 172, 57, 281, 349, 151, 187, 173, 213, 61, 282, 125, 265, 136, 314, 328, 159, 358, 90, 147, 372, 232, 300, 190, 324, 218, 360, 158, 211, 292, 243, 92, 29, 350, 362, 248, 303, 334, 250, 132, 237, 150, 2, 287, 359, 380, 222, 192, 45, 321, 42, 113, 204, 367, 206, 272, 135, 37, 105, 86, 153, 355, 10, 267, 263, 301, 182, 369, 48, 134, 210, 299, 123, 212, 91, 241, 97, 18, 55, 261, 214, 56, 356, 331, 260, 131, 348, 283, 15, 279, 203, 207, 208, 115, 305, 293, 152, 242, 315, 307, 289, 43, 53, 71, 286, 294, 278, 345, 296, 199, 167, 76, 62, 145, 74, 104, 38, 363, 205, 193, 247, 89, 357, 80, 351, 12, 127, 256, 32, 87, 335, 377, 46, 364, 200, 266, 78, 117, 121, 166, 327, 184, 354, 175, 233, 162, 230, 317, 14, 269, 16, 82, 217, 143, 110, 148, 251, 306, 196, 128, 85, 65, 343, 28, 235, 181, 155, 156, 88, 73, 275, 379, 138, 280, 40, 178, 366, 323, 244, 149, 332, 103, 209, 102, 381, 77, 236, 157, 240, 142, 371, 7, 291, 31, 347, 180, 70, 352, 98, 216, 95, 165, 376, 60, 106, 337, 245, 330, 141, 270, 116, 21, 375, 325, 368, 246, 370, 341, 79, 231, 96, 11, 94, 274, 326, 27, 228, 154, 262, 176, 304, 273, 101, 20, 99, 268, 139, 316, 319, 312, 174, 160, 24, 64, 146, 253, 107, 137, 67, 313, 188, 171, 23, 383, 114, 239, 353, 9, 19, 164, 5, 185, 320, 130, 75, 0, 221, 100, 49, 63, 346, 39, 47, 277, 66, 191, 54, 122, 258, 276, 284, 51, 234, 33, 344, 34, 378, 25, 50, 118, 111, 202, 302, 81, 35, 259, 69, 373, 361, 8, 124, 197, 336, 52, 311, 288, 215, 58, 177, 342, 41, 219, 112, 340, 318, 13, 72, 374, 59, 195, 194, 224, 238, 298, 365, 333, 1] + 49: [302, 255, 5, 336, 245, 170, 163, 295, 98, 239, 71, 124, 154, 232, 167, 171, 217, 68, 375, 337, 356, 64, 123, 191, 321, 278, 148, 76, 306, 333, 48, 137, 138, 189, 273, 43, 62, 117, 118, 228, 354, 13, 147, 350, 263, 188, 38, 317, 297, 83, 181, 110, 177, 246, 45, 164, 254, 16, 365, 222, 82, 36, 345, 69, 49, 283, 145, 27, 42, 18, 258, 26, 253, 37, 158, 261, 262, 286, 331, 8, 205, 17, 140, 323, 301, 238, 156, 211, 355, 361, 364, 166, 96, 287, 288, 219, 126, 149, 150, 270, 259, 260, 187, 131, 58, 28, 56, 224, 225, 327, 351, 47, 130, 113, 256, 51, 229, 152, 207, 373, 374, 197, 234, 25, 276, 133, 74, 314, 122, 80, 368, 34, 77, 308, 244, 320, 31, 192, 326, 357, 159, 334, 369, 50, 79, 184, 136, 241, 39, 218, 329, 100, 73, 1, 223, 127, 81, 33, 359, 72, 0, 285, 88, 104, 341, 281, 215, 24, 360, 233, 257, 78, 335, 199, 269, 95, 144, 54, 86, 112, 376, 330, 338, 378, 119, 206, 125, 316, 162, 231, 11, 349, 272, 210, 304, 353, 90, 91, 344, 298, 121, 310, 196, 290, 303, 371, 178, 226, 358, 84, 108, 315, 7, 21, 230, 92, 6, 251, 346, 293, 40, 289, 194, 264, 328, 340, 277, 29, 55, 173, 309, 53, 61, 282, 182, 343, 102, 266, 32, 129, 268, 193, 319, 116, 208, 243, 132, 109, 235, 382, 318, 128, 139, 179, 227, 85, 12, 284, 348, 111, 372, 249, 141, 347, 75, 242, 294, 213, 201, 107, 220, 94, 195, 221, 23, 89, 20, 67, 103, 114, 204, 143, 22, 57, 165, 161, 252, 135, 155, 9, 15, 115, 275, 60, 120, 183, 250, 362, 198, 214, 216, 10, 134, 280, 46, 342, 185, 168, 363, 236, 247, 370, 305, 271, 65, 105, 180, 325, 312, 332, 169, 174, 311, 41, 248, 267, 142, 339, 2, 151, 324, 274, 367, 99, 291, 300, 19, 202, 59, 212, 44, 380, 381, 30, 366, 63, 70, 160, 146, 352, 172, 14, 106, 101, 52, 377, 237, 265, 203, 3, 157, 313, 209, 66, 153, 292, 186, 175, 97, 279, 240, 4, 296, 35, 307, 176, 200, 190, 299, 93, 87, 379, 383, 322] + 50: [86, 14, 146, 327, 22, 253, 196, 313, 102, 342, 151, 2, 278, 362, 153, 349, 318, 162, 13, 157, 161, 118, 111, 345, 268, 67, 110, 296, 295, 224, 122, 48, 348, 169, 234, 322, 172, 5, 124, 7, 248, 147, 77, 72, 261, 364, 182, 365, 29, 128, 38, 287, 199, 79, 16, 164, 71, 240, 197, 382, 91, 108, 37, 177, 63, 141, 11, 324, 265, 266, 44, 84, 100, 231, 333, 125, 45, 331, 213, 274, 117, 116, 252, 81, 107, 99, 127, 204, 12, 354, 171, 61, 173, 309, 254, 136, 316, 129, 229, 366, 220, 51, 155, 96, 19, 263, 174, 371, 242, 113, 135, 123, 106, 148, 65, 101, 216, 31, 368, 230, 109, 3, 97, 373, 192, 314, 159, 298, 75, 154, 250, 139, 255, 143, 236, 170, 351, 293, 98, 30, 207, 167, 227, 283, 49, 1, 140, 273, 62, 88, 270, 133, 380, 379, 180, 310, 145, 356, 178, 288, 200, 212, 87, 346, 245, 104, 59, 332, 181, 18, 137, 35, 329, 119, 249, 187, 195, 359, 247, 92, 20, 357, 191, 281, 156, 90, 36, 60, 8, 95, 66, 23, 166, 26, 311, 218, 338, 114, 130, 24, 149, 183, 189, 303, 64, 205, 202, 292, 46, 68, 179, 83, 103, 158, 131, 222, 376, 350, 299, 367, 304, 256, 271, 6, 228, 168, 43, 326, 152, 105, 297, 4, 305, 237, 286, 176, 190, 279, 307, 241, 32, 27, 235, 370, 280, 334, 343, 275, 186, 203, 276, 267, 126, 185, 243, 339, 308, 330, 291, 21, 201, 221, 260, 257, 315, 56, 226, 284, 9, 358, 15, 363, 306, 134, 206, 378, 163, 340, 39, 337, 238, 383, 82, 40, 328, 54, 317, 374, 210, 214, 80, 262, 312, 0, 335, 132, 58, 375, 208, 94, 160, 223, 347, 344, 320, 89, 323, 76, 246, 225, 302, 264, 188, 321, 25, 233, 377, 193, 184, 301, 239, 290, 53, 73, 277, 211, 369, 42, 34, 150, 112, 209, 360, 120, 52, 50, 215, 325, 33, 361, 175, 217, 121, 353, 165, 194, 319, 74, 294, 28, 300, 259, 198, 355, 285, 381, 70, 142, 336, 55, 69, 232, 57, 352, 282, 341, 272, 41, 244, 219, 269, 85, 78, 251, 17, 289, 372, 258, 138, 10, 47, 144, 115, 93] + 51: [24, 113, 358, 95, 330, 346, 35, 161, 81, 42, 309, 50, 324, 258, 27, 242, 238, 19, 39, 77, 227, 377, 156, 202, 186, 123, 67, 203, 188, 11, 259, 281, 359, 366, 248, 141, 289, 115, 51, 93, 159, 66, 96, 367, 293, 21, 173, 348, 372, 106, 284, 211, 124, 177, 92, 292, 56, 360, 321, 368, 317, 84, 194, 298, 197, 148, 230, 70, 355, 231, 249, 261, 102, 31, 193, 300, 260, 129, 269, 143, 382, 97, 18, 204, 361, 157, 313, 191, 307, 316, 149, 215, 120, 71, 131, 226, 137, 108, 99, 176, 87, 105, 335, 299, 23, 268, 351, 34, 75, 364, 312, 89, 319, 109, 15, 4, 128, 169, 256, 255, 373, 322, 111, 356, 60, 301, 49, 247, 47, 237, 181, 59, 236, 306, 349, 379, 363, 336, 240, 158, 277, 352, 150, 327, 375, 286, 16, 64, 140, 153, 168, 216, 303, 308, 219, 91, 195, 320, 334, 315, 273, 353, 57, 354, 132, 175, 244, 291, 221, 163, 357, 136, 343, 282, 246, 20, 275, 254, 380, 252, 362, 205, 369, 328, 145, 378, 25, 296, 217, 179, 347, 326, 241, 229, 6, 228, 32, 235, 26, 46, 331, 86, 280, 370, 10, 69, 383, 114, 30, 43, 125, 257, 305, 63, 29, 121, 37, 126, 73, 72, 65, 283, 314, 5, 183, 341, 192, 79, 41, 239, 297, 381, 340, 55, 1, 83, 376, 187, 38, 151, 210, 82, 371, 122, 152, 146, 253, 220, 212, 142, 190, 207, 48, 224, 100, 54, 276, 53, 167, 272, 154, 185, 36, 2, 110, 144, 189, 264, 62, 304, 263, 8, 285, 198, 222, 90, 3, 265, 196, 223, 80, 112, 184, 14, 165, 318, 117, 138, 200, 337, 178, 182, 40, 250, 103, 294, 344, 345, 279, 251, 33, 311, 365, 45, 333, 58, 68, 9, 174, 302, 225, 139, 323, 342, 206, 52, 270, 233, 130, 118, 245, 172, 209, 310, 164, 234, 213, 338, 288, 232, 218, 88, 199, 274, 22, 76, 208, 17, 101, 180, 104, 325, 134, 13, 85, 107, 329, 119, 166, 171, 332, 170, 78, 61, 116, 28, 0, 133, 267, 290, 12, 287, 98, 127, 135, 295, 243, 374, 271, 74, 162, 160, 44, 262, 94, 350, 278, 339, 214, 7, 201, 266, 147, 155] + 52: [381, 323, 298, 60, 269, 98, 379, 222, 167, 280, 53, 213, 308, 277, 20, 330, 380, 313, 304, 11, 12, 360, 212, 348, 99, 289, 82, 33, 168, 50, 325, 5, 315, 292, 361, 287, 326, 177, 148, 188, 141, 18, 135, 255, 317, 240, 176, 180, 334, 130, 34, 346, 44, 252, 133, 256, 223, 116, 115, 333, 93, 242, 238, 198, 143, 343, 138, 314, 373, 14, 370, 375, 102, 267, 271, 7, 260, 266, 291, 74, 178, 367, 113, 39, 45, 258, 224, 49, 303, 353, 232, 42, 159, 162, 106, 247, 107, 320, 290, 90, 357, 355, 70, 127, 83, 227, 243, 72, 66, 119, 58, 189, 318, 111, 226, 158, 9, 202, 218, 204, 157, 231, 246, 211, 41, 219, 87, 88, 117, 173, 369, 76, 324, 161, 89, 207, 47, 118, 43, 274, 79, 19, 199, 16, 71, 120, 151, 345, 172, 26, 64, 123, 262, 281, 6, 153, 91, 40, 365, 220, 150, 278, 126, 112, 185, 65, 264, 46, 104, 307, 31, 321, 78, 10, 54, 309, 105, 124, 134, 331, 55, 306, 294, 21, 179, 342, 299, 302, 187, 265, 272, 305, 165, 92, 182, 174, 13, 140, 354, 35, 166, 341, 301, 154, 374, 270, 201, 139, 51, 344, 81, 147, 36, 15, 295, 175, 97, 121, 63, 206, 183, 259, 349, 171, 285, 30, 146, 17, 69, 225, 2, 84, 230, 197, 114, 275, 145, 122, 376, 38, 283, 327, 371, 195, 184, 245, 210, 257, 339, 296, 364, 3, 279, 253, 352, 383, 152, 368, 322, 340, 276, 261, 338, 94, 103, 186, 217, 214, 77, 68, 192, 32, 37, 191, 359, 300, 319, 170, 235, 57, 194, 100, 356, 358, 363, 329, 366, 164, 137, 52, 216, 73, 193, 62, 80, 236, 155, 221, 160, 29, 351, 125, 27, 284, 59, 129, 28, 335, 132, 297, 0, 254, 336, 205, 61, 362, 169, 382, 128, 149, 312, 286, 23, 248, 22, 337, 268, 237, 251, 136, 101, 4, 181, 131, 142, 1, 239, 25, 372, 85, 163, 288, 196, 95, 108, 378, 233, 75, 377, 316, 350, 110, 244, 249, 156, 96, 229, 203, 144, 24, 209, 48, 332, 250, 263, 190, 56, 8, 228, 234, 215, 67, 328, 208, 109, 200, 282, 347, 310, 311, 293, 273, 241, 86] + 53: [110, 63, 184, 356, 165, 244, 314, 377, 265, 22, 252, 56, 90, 18, 140, 153, 167, 312, 316, 185, 84, 114, 210, 272, 240, 142, 42, 6, 144, 221, 253, 262, 172, 335, 315, 122, 369, 158, 79, 40, 254, 264, 100, 147, 366, 124, 204, 290, 73, 297, 365, 180, 194, 368, 171, 25, 183, 277, 47, 67, 120, 166, 236, 220, 195, 80, 199, 193, 34, 27, 274, 307, 38, 68, 292, 92, 309, 81, 20, 62, 383, 197, 149, 226, 233, 370, 333, 299, 258, 246, 82, 310, 267, 211, 71, 58, 16, 8, 280, 118, 202, 26, 284, 88, 141, 37, 176, 293, 196, 276, 278, 232, 45, 137, 83, 39, 93, 190, 187, 223, 317, 75, 173, 155, 156, 157, 125, 127, 12, 338, 235, 150, 143, 216, 321, 30, 182, 54, 367, 177, 298, 208, 192, 304, 13, 217, 164, 227, 209, 268, 107, 5, 200, 116, 49, 89, 359, 178, 169, 255, 123, 55, 380, 48, 85, 374, 212, 355, 350, 308, 320, 224, 145, 86, 31, 95, 139, 361, 60, 10, 146, 115, 174, 96, 294, 21, 201, 283, 106, 260, 161, 99, 23, 336, 129, 379, 302, 261, 103, 325, 94, 112, 373, 281, 328, 36, 57, 215, 218, 87, 154, 51, 342, 104, 288, 245, 334, 230, 247, 179, 228, 162, 130, 219, 256, 231, 300, 15, 282, 339, 375, 207, 59, 275, 7, 305, 113, 263, 148, 11, 287, 186, 238, 46, 279, 291, 134, 326, 69, 205, 78, 349, 357, 128, 189, 28, 364, 337, 53, 9, 17, 135, 41, 213, 296, 4, 270, 363, 170, 332, 271, 3, 168, 188, 313, 72, 102, 248, 371, 117, 133, 97, 131, 249, 64, 382, 91, 286, 109, 32, 66, 345, 266, 376, 121, 269, 70, 77, 324, 126, 175, 35, 65, 306, 242, 257, 108, 43, 74, 163, 61, 191, 354, 330, 203, 322, 360, 225, 119, 24, 381, 151, 378, 159, 353, 111, 105, 19, 251, 234, 352, 323, 301, 303, 181, 318, 311, 250, 243, 273, 132, 237, 285, 319, 346, 152, 239, 76, 44, 340, 327, 136, 138, 101, 289, 331, 348, 372, 343, 344, 347, 295, 222, 214, 160, 351, 50, 241, 329, 341, 52, 259, 362, 2, 206, 358, 198, 0, 14, 29, 1, 229, 98, 33] + 54: [363, 357, 190, 239, 55, 61, 326, 81, 351, 243, 6, 233, 362, 104, 252, 383, 127, 205, 148, 232, 320, 336, 361, 378, 370, 72, 345, 118, 332, 115, 198, 185, 125, 365, 73, 111, 364, 141, 194, 235, 331, 60, 303, 188, 342, 354, 0, 36, 294, 267, 20, 282, 119, 3, 47, 161, 5, 217, 186, 135, 255, 209, 18, 273, 213, 110, 79, 263, 210, 222, 349, 13, 339, 71, 150, 306, 221, 7, 121, 218, 335, 34, 381, 131, 144, 107, 146, 262, 371, 211, 236, 227, 307, 241, 78, 106, 155, 253, 82, 174, 330, 314, 10, 310, 50, 152, 140, 368, 171, 327, 33, 300, 259, 219, 268, 66, 51, 379, 212, 160, 164, 296, 257, 53, 187, 180, 101, 333, 84, 265, 103, 355, 278, 293, 96, 244, 139, 15, 367, 108, 374, 350, 165, 123, 284, 153, 216, 8, 290, 117, 317, 70, 215, 94, 272, 237, 100, 373, 62, 167, 316, 308, 224, 269, 248, 65, 134, 358, 283, 380, 287, 172, 341, 26, 181, 132, 196, 321, 92, 184, 246, 264, 304, 377, 238, 245, 59, 249, 56, 149, 256, 193, 42, 337, 189, 30, 176, 309, 58, 288, 113, 91, 24, 21, 1, 366, 179, 114, 23, 12, 74, 274, 199, 285, 37, 207, 258, 295, 83, 369, 93, 311, 329, 276, 129, 323, 382, 157, 318, 68, 137, 151, 43, 63, 145, 289, 270, 360, 126, 297, 11, 95, 338, 214, 44, 87, 86, 41, 158, 225, 54, 170, 177, 298, 201, 353, 275, 88, 168, 31, 46, 35, 169, 130, 356, 286, 147, 133, 208, 25, 16, 375, 197, 45, 159, 204, 67, 48, 313, 166, 9, 251, 178, 206, 102, 116, 77, 19, 223, 299, 39, 98, 120, 305, 372, 99, 64, 220, 182, 312, 315, 163, 22, 173, 128, 271, 247, 261, 203, 154, 138, 200, 231, 230, 90, 250, 254, 57, 75, 143, 277, 124, 109, 97, 302, 85, 122, 195, 191, 334, 14, 29, 226, 292, 291, 202, 240, 280, 328, 80, 281, 40, 27, 156, 32, 325, 142, 376, 183, 69, 136, 347, 175, 192, 234, 52, 343, 4, 2, 340, 348, 38, 319, 260, 324, 352, 228, 76, 322, 17, 279, 105, 49, 301, 242, 162, 266, 229, 346, 89, 359, 28, 112, 344] + 55: [154, 16, 247, 49, 99, 300, 337, 348, 231, 361, 122, 157, 149, 127, 365, 168, 58, 248, 155, 244, 351, 186, 215, 34, 174, 7, 227, 363, 134, 44, 26, 163, 375, 178, 140, 207, 87, 232, 373, 336, 275, 261, 175, 362, 148, 51, 69, 305, 56, 113, 234, 307, 304, 293, 331, 271, 112, 105, 265, 54, 318, 85, 31, 188, 356, 243, 66, 242, 141, 108, 167, 283, 202, 326, 4, 111, 382, 223, 320, 0, 156, 276, 378, 306, 332, 52, 101, 133, 367, 55, 104, 369, 169, 142, 229, 310, 88, 76, 309, 238, 78, 249, 272, 138, 17, 325, 236, 343, 257, 358, 314, 342, 15, 372, 50, 145, 181, 36, 345, 109, 81, 115, 151, 311, 73, 40, 226, 368, 189, 350, 217, 42, 263, 277, 302, 237, 253, 30, 107, 374, 282, 334, 24, 285, 98, 269, 355, 218, 211, 301, 12, 164, 259, 170, 380, 47, 119, 287, 214, 153, 97, 321, 315, 324, 330, 21, 100, 196, 20, 29, 256, 89, 192, 2, 179, 213, 172, 46, 377, 246, 10, 23, 313, 1, 219, 126, 125, 64, 364, 195, 292, 144, 11, 130, 235, 357, 267, 200, 71, 216, 344, 91, 371, 187, 241, 171, 296, 103, 37, 297, 354, 13, 92, 110, 379, 250, 146, 180, 270, 62, 328, 291, 176, 366, 206, 284, 251, 79, 19, 346, 264, 82, 268, 319, 150, 68, 191, 266, 9, 45, 308, 323, 43, 370, 121, 70, 333, 86, 233, 352, 27, 240, 184, 128, 63, 198, 359, 35, 120, 139, 295, 32, 158, 204, 340, 74, 327, 182, 41, 132, 279, 349, 5, 102, 224, 360, 322, 255, 117, 212, 225, 93, 137, 162, 280, 83, 59, 222, 116, 347, 28, 39, 303, 106, 3, 290, 197, 14, 289, 286, 185, 254, 22, 61, 335, 201, 338, 183, 77, 288, 25, 205, 80, 48, 152, 165, 353, 199, 8, 312, 383, 190, 210, 53, 228, 84, 294, 160, 143, 118, 161, 177, 298, 57, 230, 317, 381, 90, 159, 252, 273, 65, 60, 316, 124, 274, 329, 135, 194, 278, 33, 258, 173, 38, 129, 166, 262, 75, 131, 245, 123, 299, 72, 339, 6, 341, 114, 221, 239, 95, 260, 67, 136, 193, 94, 281, 208, 209, 147, 203, 376, 220, 18, 96] + 56: [231, 9, 251, 284, 77, 207, 103, 250, 23, 317, 227, 95, 201, 375, 276, 65, 373, 337, 203, 36, 301, 173, 202, 90, 240, 88, 356, 6, 275, 80, 118, 73, 281, 374, 280, 243, 297, 134, 145, 200, 289, 60, 211, 212, 366, 74, 315, 3, 195, 107, 303, 264, 316, 2, 168, 369, 38, 105, 295, 11, 151, 368, 180, 314, 226, 367, 93, 254, 349, 377, 142, 146, 260, 167, 158, 263, 320, 332, 357, 0, 40, 324, 338, 34, 138, 364, 54, 96, 28, 372, 113, 47, 358, 247, 85, 224, 220, 8, 163, 196, 328, 302, 288, 72, 359, 10, 79, 149, 33, 129, 360, 347, 232, 16, 352, 272, 336, 174, 165, 143, 294, 177, 155, 189, 308, 41, 14, 383, 58, 312, 268, 269, 185, 221, 62, 172, 319, 29, 43, 26, 52, 45, 235, 208, 321, 100, 116, 322, 104, 71, 188, 101, 309, 148, 193, 179, 12, 270, 313, 271, 228, 330, 67, 350, 108, 59, 216, 318, 242, 279, 273, 351, 187, 46, 362, 381, 233, 160, 278, 166, 355, 209, 339, 299, 310, 379, 78, 361, 382, 261, 217, 51, 265, 147, 186, 267, 333, 48, 18, 192, 238, 69, 92, 305, 133, 255, 248, 94, 341, 175, 57, 152, 131, 215, 194, 61, 44, 304, 53, 371, 236, 39, 329, 244, 246, 27, 206, 198, 214, 164, 286, 210, 353, 274, 139, 287, 157, 68, 277, 102, 178, 75, 199, 156, 306, 346, 83, 150, 223, 182, 181, 136, 110, 153, 42, 55, 262, 124, 266, 130, 345, 370, 365, 229, 82, 5, 81, 169, 125, 161, 66, 63, 197, 285, 135, 252, 50, 17, 56, 183, 84, 293, 49, 170, 325, 7, 380, 76, 213, 30, 191, 87, 363, 140, 19, 245, 205, 342, 128, 35, 22, 123, 378, 257, 119, 307, 64, 70, 234, 326, 25, 171, 15, 259, 126, 184, 89, 99, 327, 283, 344, 376, 21, 31, 331, 127, 115, 122, 132, 4, 291, 24, 204, 239, 111, 237, 225, 340, 311, 98, 20, 162, 219, 154, 97, 300, 253, 354, 1, 13, 282, 298, 218, 323, 144, 86, 256, 335, 114, 91, 241, 106, 343, 296, 249, 117, 292, 159, 120, 334, 137, 230, 121, 141, 190, 348, 290, 112, 37, 222, 258, 176, 32, 109] + 57: [254, 344, 318, 35, 58, 179, 113, 277, 55, 225, 209, 81, 334, 236, 200, 224, 370, 264, 294, 181, 367, 301, 304, 343, 276, 220, 198, 267, 283, 88, 272, 243, 214, 183, 380, 279, 321, 98, 29, 315, 281, 194, 374, 266, 82, 336, 293, 372, 326, 342, 156, 124, 204, 79, 354, 43, 341, 83, 92, 119, 149, 23, 18, 247, 59, 357, 222, 78, 218, 278, 34, 285, 25, 11, 63, 274, 169, 22, 250, 365, 297, 46, 147, 112, 71, 369, 323, 145, 24, 188, 361, 287, 115, 258, 30, 56, 231, 93, 368, 359, 60, 180, 251, 316, 9, 20, 116, 13, 313, 90, 311, 97, 265, 168, 242, 364, 244, 212, 195, 86, 14, 157, 57, 108, 300, 345, 185, 91, 110, 177, 111, 38, 166, 199, 132, 162, 355, 87, 50, 192, 61, 340, 245, 234, 146, 154, 158, 235, 335, 150, 123, 167, 172, 249, 356, 106, 329, 337, 17, 371, 33, 4, 109, 239, 100, 291, 227, 262, 259, 74, 16, 140, 118, 197, 160, 238, 346, 152, 349, 289, 2, 196, 155, 237, 252, 0, 1, 76, 273, 189, 305, 275, 295, 302, 133, 7, 217, 317, 64, 182, 73, 42, 377, 358, 210, 352, 216, 261, 5, 378, 219, 103, 142, 52, 37, 229, 165, 125, 31, 65, 130, 117, 306, 39, 161, 173, 312, 12, 328, 296, 330, 332, 45, 255, 66, 138, 310, 221, 131, 105, 89, 170, 314, 128, 141, 350, 80, 253, 373, 298, 191, 47, 348, 190, 186, 379, 122, 270, 333, 232, 360, 327, 153, 233, 246, 320, 121, 366, 8, 127, 163, 99, 176, 27, 382, 120, 68, 351, 178, 175, 303, 307, 241, 107, 10, 129, 280, 19, 187, 286, 85, 101, 324, 151, 309, 347, 213, 51, 184, 284, 95, 282, 215, 41, 363, 126, 260, 375, 271, 206, 15, 201, 205, 72, 230, 257, 137, 6, 75, 319, 40, 228, 54, 325, 226, 3, 362, 26, 49, 32, 36, 256, 339, 114, 292, 376, 148, 331, 381, 104, 268, 84, 44, 338, 322, 96, 21, 208, 164, 353, 288, 240, 135, 248, 263, 48, 139, 193, 69, 143, 207, 94, 159, 223, 53, 174, 62, 144, 134, 70, 102, 203, 136, 171, 28, 290, 211, 299, 383, 67, 308, 269, 77, 202] + 58: [377, 3, 71, 125, 319, 321, 335, 370, 10, 108, 150, 168, 184, 280, 364, 201, 204, 340, 33, 38, 58, 134, 209, 308, 220, 178, 341, 12, 190, 187, 239, 82, 333, 63, 300, 43, 113, 8, 296, 359, 290, 349, 193, 74, 212, 158, 292, 309, 81, 344, 289, 199, 18, 16, 106, 198, 251, 62, 250, 110, 241, 21, 347, 237, 138, 100, 44, 51, 216, 174, 313, 279, 245, 79, 322, 355, 175, 6, 104, 242, 380, 25, 284, 53, 131, 253, 65, 19, 230, 142, 372, 46, 84, 304, 217, 291, 156, 61, 373, 67, 232, 47, 132, 197, 281, 244, 181, 339, 302, 144, 371, 265, 133, 170, 303, 157, 369, 286, 310, 165, 275, 76, 101, 351, 36, 85, 130, 352, 229, 207, 59, 124, 346, 231, 123, 115, 180, 243, 86, 276, 96, 15, 301, 147, 126, 151, 271, 358, 112, 141, 191, 4, 28, 314, 266, 160, 378, 154, 288, 35, 94, 294, 221, 282, 39, 32, 140, 91, 56, 249, 14, 72, 116, 342, 31, 20, 328, 55, 383, 183, 149, 357, 161, 182, 338, 5, 317, 337, 57, 34, 218, 361, 366, 77, 210, 22, 368, 324, 272, 316, 73, 202, 350, 83, 0, 172, 228, 11, 179, 360, 87, 374, 109, 145, 323, 89, 305, 248, 211, 69, 169, 295, 189, 274, 224, 381, 143, 227, 37, 176, 327, 50, 105, 24, 54, 254, 117, 153, 343, 233, 205, 263, 348, 9, 256, 45, 277, 60, 285, 375, 258, 234, 382, 186, 107, 225, 118, 297, 163, 354, 120, 159, 363, 23, 92, 114, 27, 49, 40, 196, 97, 260, 329, 331, 129, 155, 135, 13, 278, 195, 173, 102, 128, 336, 246, 137, 30, 320, 98, 298, 273, 299, 26, 68, 367, 315, 66, 48, 215, 238, 240, 148, 334, 283, 219, 213, 264, 146, 167, 17, 127, 80, 194, 122, 259, 235, 270, 376, 42, 111, 307, 139, 236, 306, 75, 379, 318, 293, 226, 64, 268, 41, 223, 52, 188, 164, 311, 312, 7, 93, 214, 262, 203, 255, 185, 1, 325, 326, 121, 208, 345, 287, 136, 261, 330, 200, 247, 78, 267, 166, 222, 177, 356, 206, 257, 152, 119, 162, 171, 192, 99, 90, 332, 362, 2, 70, 353, 29, 95, 103, 269, 88, 365, 252] + 59: [327, 371, 172, 80, 114, 164, 211, 315, 214, 344, 36, 96, 0, 246, 37, 88, 148, 312, 305, 39, 55, 235, 154, 189, 25, 33, 95, 150, 38, 112, 109, 279, 22, 19, 137, 337, 199, 206, 41, 74, 89, 166, 319, 237, 293, 139, 308, 215, 170, 182, 70, 300, 262, 58, 57, 297, 60, 142, 146, 203, 292, 333, 110, 153, 145, 296, 257, 108, 14, 87, 268, 271, 229, 113, 5, 331, 9, 309, 278, 16, 383, 245, 222, 238, 349, 48, 149, 322, 326, 106, 102, 8, 316, 77, 378, 341, 7, 73, 233, 130, 372, 266, 91, 168, 169, 29, 179, 92, 131, 32, 64, 253, 359, 249, 307, 301, 98, 125, 78, 117, 140, 332, 147, 291, 186, 281, 45, 160, 163, 248, 356, 180, 155, 61, 71, 34, 21, 132, 275, 358, 350, 121, 190, 4, 209, 65, 226, 221, 324, 382, 176, 35, 302, 373, 47, 198, 365, 107, 299, 184, 217, 210, 11, 103, 360, 375, 116, 351, 224, 234, 197, 294, 295, 243, 212, 254, 280, 252, 10, 258, 101, 2, 43, 129, 67, 183, 191, 284, 192, 56, 122, 167, 250, 352, 97, 263, 185, 178, 128, 40, 127, 174, 306, 348, 289, 242, 138, 187, 330, 277, 273, 82, 141, 171, 126, 15, 379, 346, 313, 311, 18, 377, 342, 175, 181, 225, 338, 369, 361, 362, 194, 208, 28, 93, 357, 368, 239, 244, 152, 90, 370, 66, 30, 251, 213, 26, 196, 81, 62, 320, 158, 69, 353, 156, 380, 84, 134, 336, 119, 216, 276, 264, 345, 188, 94, 220, 75, 204, 76, 328, 290, 223, 100, 376, 236, 339, 228, 51, 193, 50, 17, 3, 201, 283, 282, 367, 83, 111, 195, 340, 285, 374, 42, 317, 133, 161, 159, 218, 329, 177, 259, 68, 27, 135, 207, 49, 321, 165, 318, 205, 347, 79, 202, 200, 46, 247, 31, 151, 255, 85, 334, 162, 355, 120, 6, 314, 287, 231, 157, 44, 118, 241, 219, 270, 72, 323, 260, 173, 105, 298, 265, 144, 86, 304, 267, 143, 54, 366, 354, 123, 63, 325, 343, 261, 53, 303, 1, 272, 104, 310, 59, 363, 256, 381, 288, 23, 232, 12, 364, 20, 286, 269, 13, 274, 227, 136, 124, 24, 115, 230, 240, 335, 99, 52] + 60: [313, 309, 161, 253, 248, 93, 11, 305, 90, 66, 369, 371, 363, 287, 216, 111, 249, 71, 77, 62, 154, 288, 22, 355, 124, 34, 119, 232, 364, 150, 37, 80, 342, 53, 273, 314, 225, 383, 324, 261, 298, 306, 138, 87, 157, 207, 375, 201, 351, 167, 208, 337, 15, 203, 359, 255, 269, 9, 377, 21, 33, 95, 368, 270, 251, 145, 211, 326, 177, 109, 214, 348, 123, 347, 120, 74, 290, 206, 271, 152, 114, 92, 38, 134, 196, 182, 219, 307, 252, 54, 378, 292, 275, 10, 373, 299, 257, 243, 222, 181, 294, 112, 131, 169, 84, 358, 88, 56, 233, 91, 130, 256, 282, 361, 42, 68, 301, 73, 218, 317, 16, 170, 39, 65, 234, 180, 278, 172, 336, 6, 332, 304, 346, 200, 192, 286, 105, 13, 258, 29, 350, 4, 44, 63, 268, 121, 343, 247, 28, 353, 213, 235, 75, 221, 344, 3, 25, 142, 100, 296, 148, 67, 328, 374, 175, 17, 173, 210, 381, 236, 244, 230, 2, 49, 98, 59, 140, 283, 240, 264, 26, 31, 30, 349, 315, 202, 320, 224, 291, 72, 103, 101, 158, 94, 303, 194, 155, 319, 354, 223, 241, 362, 259, 267, 176, 52, 187, 370, 64, 125, 330, 198, 229, 43, 227, 308, 220, 102, 266, 316, 297, 76, 365, 245, 263, 20, 183, 237, 274, 70, 285, 322, 380, 376, 45, 280, 356, 135, 289, 89, 47, 284, 115, 212, 185, 239, 311, 117, 55, 69, 318, 5, 61, 141, 7, 341, 12, 333, 178, 19, 97, 254, 367, 136, 139, 310, 195, 48, 334, 108, 166, 272, 82, 277, 1, 382, 279, 107, 85, 357, 14, 338, 127, 41, 160, 86, 295, 118, 60, 265, 129, 78, 302, 327, 329, 231, 99, 189, 132, 24, 8, 360, 293, 106, 144, 156, 262, 50, 379, 159, 340, 321, 126, 174, 260, 113, 312, 205, 193, 184, 27, 151, 51, 162, 57, 171, 149, 79, 276, 168, 23, 104, 204, 165, 32, 352, 83, 40, 335, 339, 153, 366, 345, 143, 238, 58, 217, 110, 215, 323, 250, 300, 331, 197, 209, 137, 147, 281, 163, 226, 133, 199, 128, 372, 81, 35, 46, 122, 246, 0, 36, 188, 116, 186, 146, 190, 18, 325, 228, 179, 96, 191, 164, 242] + 61: [221, 178, 353, 119, 333, 243, 224, 366, 47, 23, 38, 273, 295, 132, 130, 91, 13, 201, 35, 254, 76, 116, 319, 58, 127, 205, 45, 294, 291, 31, 92, 316, 121, 245, 290, 153, 242, 26, 359, 63, 123, 185, 146, 279, 41, 343, 155, 0, 179, 108, 117, 218, 143, 34, 282, 65, 83, 379, 97, 173, 9, 22, 14, 106, 101, 99, 138, 339, 382, 183, 356, 149, 338, 360, 323, 21, 215, 177, 133, 258, 29, 2, 368, 137, 214, 196, 71, 193, 54, 24, 314, 56, 60, 376, 171, 208, 61, 303, 165, 1, 68, 327, 349, 337, 90, 252, 69, 249, 78, 365, 67, 81, 244, 163, 111, 272, 318, 277, 229, 40, 73, 231, 351, 380, 371, 288, 181, 228, 233, 113, 3, 102, 292, 125, 240, 124, 274, 310, 331, 305, 347, 281, 217, 357, 93, 107, 247, 184, 51, 192, 159, 75, 297, 82, 308, 202, 378, 37, 79, 284, 152, 301, 358, 265, 105, 131, 62, 96, 211, 216, 326, 264, 278, 66, 42, 43, 150, 27, 260, 147, 139, 306, 141, 317, 352, 271, 246, 293, 220, 377, 16, 296, 46, 88, 170, 209, 372, 112, 286, 261, 77, 190, 336, 200, 259, 174, 126, 342, 148, 255, 7, 95, 222, 226, 266, 334, 299, 86, 157, 59, 12, 238, 110, 160, 348, 114, 374, 207, 72, 346, 340, 4, 311, 239, 49, 180, 276, 64, 320, 128, 197, 161, 172, 270, 17, 142, 115, 169, 191, 300, 304, 328, 154, 30, 283, 194, 298, 6, 251, 164, 250, 175, 50, 189, 381, 307, 230, 241, 330, 198, 168, 309, 20, 84, 109, 52, 186, 341, 219, 204, 98, 364, 74, 182, 236, 355, 267, 321, 100, 263, 5, 10, 363, 55, 188, 227, 361, 268, 257, 206, 89, 275, 85, 199, 345, 25, 232, 212, 256, 39, 134, 94, 36, 248, 136, 203, 103, 213, 57, 156, 362, 166, 313, 158, 33, 375, 225, 144, 370, 354, 312, 28, 140, 151, 367, 53, 167, 324, 335, 122, 253, 237, 129, 15, 118, 135, 329, 145, 383, 8, 80, 332, 70, 187, 262, 373, 302, 285, 325, 350, 315, 344, 120, 19, 195, 235, 369, 32, 87, 44, 11, 104, 322, 280, 269, 176, 162, 18, 234, 48, 223, 289, 210, 287] +layer_updates_per_iter: 0 +num_slots: 384 diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-moe-load-balancer-configs/moe_load_balancer_gen_ep32_slots384.yaml b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-moe-load-balancer-configs/moe_load_balancer_gen_ep32_slots384.yaml new file mode 100644 index 0000000000..262c8042d3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-moe-load-balancer-configs/moe_load_balancer_gen_ep32_slots384.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [165, 327, 22, 354, 123, 257, 180, 263, 53, 250, 120, 61, 111, 232, 55, 10, 31, 213, 192, 145, 340, 132, 88, 185, 378, 29, 282, 156, 148, 172, 369, 67, 206, 164, 168, 303, 166, 104, 272, 72, 114, 358, 249, 147, 252, 266, 304, 247, 256, 347, 198, 284, 368, 375, 331, 267, 350, 125, 142, 110, 74, 188, 63, 117, 261, 66, 365, 326, 229, 357, 24, 373, 222, 332, 139, 34, 200, 39, 269, 299, 40, 2, 121, 274, 356, 190, 343, 205, 36, 89, 108, 25, 210, 301, 339, 370, 236, 241, 255, 78, 297, 309, 103, 100, 47, 175, 189, 201, 245, 346, 48, 359, 186, 160, 4, 97, 20, 41, 271, 171, 371, 158, 321, 143, 90, 27, 317, 1, 50, 12, 92, 13, 225, 230, 289, 141, 124, 268, 136, 238, 6, 353, 23, 277, 254, 193, 218, 336, 57, 77, 214, 215, 260, 380, 298, 118, 135, 207, 157, 360, 109, 149, 335, 211, 342, 306, 18, 58, 279, 76, 80, 349, 56, 337, 227, 296, 362, 315, 167, 310, 45, 264, 242, 126, 233, 208, 244, 59, 237, 115, 251, 288, 60, 235, 62, 209, 43, 170, 32, 366, 14, 352, 69, 226, 79, 177, 204, 144, 183, 7, 85, 372, 82, 98, 319, 8, 17, 348, 30, 280, 307, 300, 262, 75, 83, 286, 381, 344, 95, 162, 202, 3, 181, 28, 314, 363, 127, 294, 194, 291, 134, 155, 81, 333, 131, 112, 38, 146, 328, 305, 231, 228, 184, 86, 153, 176, 341, 265, 308, 275, 320, 316, 239, 281, 49, 383, 21, 16, 33, 248, 234, 240, 93, 102, 285, 329, 203, 119, 35, 37, 196, 311, 377, 216, 195, 64, 191, 273, 217, 322, 295, 318, 54, 44, 199, 46, 382, 338, 367, 325, 330, 345, 169, 70, 101, 94, 223, 19, 106, 253, 9, 374, 173, 351, 122, 276, 313, 324, 87, 116, 292, 96, 334, 323, 224, 84, 312, 65, 243, 138, 52, 137, 73, 154, 220, 287, 290, 159, 182, 376, 11, 361, 187, 99, 379, 212, 179, 355, 259, 302, 26, 278, 105, 129, 91, 140, 221, 15, 219, 130, 0, 71, 51, 293, 107, 258, 133, 178, 270, 42, 113, 128, 150, 151, 163, 68, 283, 174, 5, 246, 197, 161, 152, 364] + 1: [163, 0, 29, 355, 176, 205, 318, 293, 119, 253, 117, 255, 319, 39, 24, 78, 143, 133, 2, 290, 77, 316, 339, 100, 365, 131, 248, 89, 261, 376, 259, 57, 181, 220, 45, 265, 42, 40, 23, 32, 280, 171, 204, 165, 113, 372, 191, 15, 201, 21, 250, 173, 335, 239, 305, 321, 268, 129, 16, 363, 174, 378, 56, 86, 142, 202, 222, 154, 8, 331, 155, 327, 22, 329, 299, 71, 194, 38, 348, 33, 302, 371, 322, 215, 137, 254, 244, 260, 92, 360, 170, 136, 323, 246, 124, 310, 180, 291, 300, 275, 162, 69, 79, 229, 13, 375, 343, 368, 158, 196, 294, 211, 84, 306, 313, 295, 94, 279, 128, 34, 277, 43, 58, 287, 108, 264, 140, 175, 96, 130, 361, 345, 382, 152, 334, 145, 288, 88, 362, 184, 227, 369, 252, 336, 91, 50, 63, 37, 212, 116, 298, 156, 352, 357, 221, 312, 72, 263, 105, 9, 146, 93, 76, 17, 97, 83, 135, 112, 169, 149, 73, 269, 187, 381, 7, 115, 104, 148, 217, 199, 177, 150, 110, 303, 166, 203, 121, 107, 311, 190, 209, 267, 167, 126, 328, 195, 197, 289, 122, 296, 26, 314, 132, 157, 127, 85, 41, 273, 218, 358, 164, 19, 350, 103, 66, 304, 347, 80, 123, 138, 301, 282, 189, 14, 51, 27, 208, 354, 67, 106, 210, 59, 99, 228, 256, 326, 11, 356, 10, 257, 377, 44, 70, 243, 74, 206, 237, 332, 315, 333, 373, 258, 3, 18, 325, 46, 285, 47, 231, 20, 48, 232, 151, 374, 182, 118, 75, 320, 274, 213, 98, 370, 284, 54, 340, 341, 198, 308, 179, 383, 266, 214, 61, 286, 87, 240, 207, 64, 120, 31, 5, 283, 216, 52, 60, 65, 186, 262, 160, 337, 225, 235, 270, 224, 281, 4, 95, 139, 324, 353, 193, 159, 111, 1, 366, 25, 245, 53, 102, 161, 36, 226, 367, 349, 82, 200, 192, 233, 317, 219, 101, 249, 35, 12, 90, 276, 272, 81, 6, 185, 234, 114, 236, 364, 230, 344, 188, 144, 292, 251, 30, 351, 125, 223, 147, 338, 172, 49, 297, 359, 134, 141, 168, 153, 247, 379, 278, 330, 68, 346, 183, 307, 178, 380, 241, 271, 242, 109, 342, 238, 62, 55, 309, 28] + 2: [143, 284, 325, 79, 92, 46, 136, 239, 371, 48, 228, 379, 248, 97, 31, 323, 55, 332, 154, 275, 373, 74, 326, 381, 53, 350, 51, 285, 224, 349, 305, 77, 292, 177, 289, 374, 43, 111, 45, 30, 263, 360, 106, 23, 329, 222, 315, 258, 132, 149, 118, 235, 126, 163, 9, 362, 99, 98, 287, 366, 112, 60, 245, 82, 262, 277, 38, 94, 95, 164, 202, 229, 108, 75, 278, 24, 68, 253, 256, 152, 216, 382, 282, 40, 313, 33, 70, 340, 90, 120, 181, 179, 144, 267, 237, 190, 215, 197, 286, 279, 7, 128, 160, 176, 141, 8, 354, 199, 89, 34, 73, 200, 17, 201, 15, 365, 331, 205, 260, 330, 194, 247, 280, 63, 170, 58, 186, 150, 264, 167, 240, 161, 93, 125, 56, 217, 299, 383, 169, 104, 207, 114, 13, 300, 115, 151, 131, 146, 206, 21, 309, 29, 376, 338, 353, 363, 223, 100, 66, 328, 140, 20, 193, 318, 178, 335, 369, 375, 3, 189, 355, 83, 130, 361, 175, 268, 52, 183, 317, 255, 295, 321, 157, 274, 344, 187, 348, 257, 272, 251, 214, 246, 212, 137, 173, 26, 204, 327, 308, 129, 25, 226, 72, 316, 242, 221, 166, 22, 294, 210, 41, 276, 219, 291, 117, 358, 232, 5, 124, 368, 165, 320, 65, 364, 0, 359, 345, 127, 290, 259, 191, 324, 174, 67, 148, 49, 47, 319, 147, 249, 19, 81, 322, 296, 61, 203, 76, 121, 238, 352, 310, 32, 62, 1, 84, 172, 342, 18, 171, 250, 134, 302, 378, 119, 168, 298, 155, 105, 311, 293, 265, 372, 2, 57, 336, 301, 91, 28, 192, 271, 209, 356, 241, 69, 36, 54, 182, 337, 377, 102, 109, 220, 6, 273, 16, 116, 195, 71, 158, 339, 86, 343, 110, 133, 85, 156, 10, 254, 281, 107, 307, 270, 304, 14, 101, 42, 303, 27, 236, 142, 4, 233, 234, 64, 314, 357, 78, 184, 11, 269, 145, 103, 211, 288, 113, 80, 139, 351, 185, 306, 159, 87, 138, 12, 346, 370, 59, 208, 122, 341, 153, 135, 196, 347, 198, 39, 367, 231, 37, 380, 180, 225, 162, 44, 244, 50, 297, 312, 334, 243, 227, 266, 230, 96, 213, 123, 333, 188, 218, 252, 35, 261, 283, 88] + 3: [66, 199, 278, 292, 303, 59, 311, 47, 222, 38, 301, 370, 34, 17, 247, 332, 335, 52, 9, 233, 257, 146, 324, 198, 310, 43, 165, 93, 234, 148, 352, 244, 152, 167, 226, 5, 260, 328, 15, 309, 121, 147, 124, 315, 173, 185, 341, 371, 377, 98, 270, 237, 367, 101, 134, 330, 175, 320, 326, 193, 365, 161, 181, 327, 160, 232, 362, 141, 253, 265, 349, 33, 319, 29, 229, 217, 246, 361, 153, 45, 54, 127, 4, 347, 39, 239, 364, 374, 166, 44, 249, 282, 159, 90, 272, 60, 22, 138, 118, 277, 200, 113, 187, 269, 220, 75, 308, 69, 79, 211, 119, 154, 204, 129, 286, 373, 258, 210, 368, 23, 302, 238, 325, 21, 288, 224, 266, 322, 130, 179, 143, 1, 40, 26, 189, 223, 99, 218, 307, 216, 381, 285, 338, 95, 18, 337, 250, 36, 142, 375, 105, 276, 106, 190, 372, 89, 27, 158, 102, 231, 12, 203, 103, 48, 353, 85, 183, 205, 163, 318, 201, 122, 240, 32, 299, 280, 96, 356, 245, 241, 3, 24, 186, 37, 64, 359, 108, 273, 115, 83, 354, 16, 194, 363, 139, 10, 11, 172, 41, 65, 248, 376, 0, 378, 313, 151, 339, 267, 290, 295, 126, 117, 340, 123, 133, 107, 263, 348, 215, 227, 25, 289, 274, 135, 306, 61, 206, 191, 71, 157, 137, 58, 182, 162, 300, 355, 196, 294, 28, 251, 46, 62, 42, 2, 219, 8, 345, 169, 366, 91, 254, 379, 275, 80, 331, 358, 316, 84, 176, 112, 100, 7, 86, 323, 35, 333, 6, 164, 177, 109, 155, 57, 214, 291, 346, 242, 230, 261, 213, 342, 174, 78, 111, 235, 76, 70, 228, 284, 209, 97, 50, 120, 296, 87, 243, 94, 82, 188, 110, 360, 314, 192, 180, 336, 74, 136, 297, 197, 144, 298, 283, 116, 14, 208, 252, 170, 195, 271, 92, 344, 383, 305, 68, 329, 171, 56, 81, 343, 304, 279, 114, 53, 268, 67, 72, 73, 51, 13, 145, 207, 287, 357, 312, 256, 125, 128, 132, 88, 31, 131, 49, 168, 77, 382, 351, 236, 30, 184, 293, 140, 350, 225, 334, 178, 221, 202, 262, 380, 104, 20, 321, 369, 19, 317, 150, 63, 212, 264, 149, 55, 255, 281, 259, 156] + 4: [343, 67, 321, 90, 336, 107, 182, 277, 282, 305, 332, 351, 331, 79, 128, 231, 242, 272, 338, 287, 58, 63, 104, 120, 3, 341, 137, 51, 40, 374, 368, 174, 20, 192, 208, 233, 271, 75, 288, 234, 249, 96, 293, 270, 227, 119, 225, 17, 315, 101, 260, 307, 269, 370, 175, 127, 279, 202, 154, 2, 214, 365, 28, 141, 359, 80, 178, 111, 311, 373, 299, 337, 209, 325, 43, 187, 252, 64, 84, 103, 130, 1, 11, 382, 296, 86, 361, 204, 159, 14, 284, 349, 223, 205, 224, 195, 371, 171, 347, 83, 177, 65, 261, 328, 26, 286, 380, 144, 23, 150, 167, 218, 317, 133, 132, 367, 157, 355, 173, 366, 15, 105, 155, 57, 369, 85, 323, 310, 29, 201, 48, 91, 95, 257, 298, 314, 8, 38, 60, 46, 206, 327, 378, 248, 226, 45, 197, 4, 7, 276, 379, 232, 184, 295, 362, 191, 319, 5, 377, 106, 161, 116, 53, 322, 52, 237, 112, 164, 87, 19, 36, 196, 339, 358, 185, 108, 121, 122, 123, 238, 194, 376, 308, 76, 82, 207, 149, 125, 303, 170, 115, 102, 273, 281, 6, 139, 73, 0, 356, 100, 220, 31, 156, 236, 117, 88, 165, 145, 251, 99, 168, 94, 292, 109, 213, 266, 142, 239, 143, 32, 49, 183, 297, 131, 50, 140, 255, 363, 148, 9, 241, 61, 280, 221, 189, 354, 176, 181, 10, 285, 254, 172, 136, 77, 68, 42, 360, 135, 309, 268, 126, 301, 152, 228, 348, 200, 344, 264, 72, 313, 263, 291, 203, 245, 188, 16, 302, 275, 217, 243, 330, 25, 39, 289, 353, 55, 34, 193, 253, 235, 44, 179, 33, 70, 66, 304, 163, 162, 340, 320, 345, 372, 312, 153, 352, 240, 92, 334, 316, 247, 18, 250, 56, 113, 22, 244, 114, 262, 267, 118, 21, 30, 37, 259, 13, 41, 35, 54, 256, 383, 294, 147, 129, 134, 12, 219, 274, 357, 333, 324, 62, 74, 346, 210, 186, 326, 375, 350, 342, 318, 198, 290, 166, 158, 306, 211, 97, 265, 124, 24, 329, 364, 160, 278, 169, 229, 78, 146, 215, 89, 199, 258, 59, 246, 110, 335, 98, 216, 230, 180, 81, 212, 222, 300, 47, 151, 27, 283, 93, 69, 138, 71, 381, 190] + 5: [61, 207, 144, 300, 236, 129, 340, 226, 286, 177, 354, 302, 213, 37, 295, 56, 284, 363, 316, 85, 142, 179, 118, 303, 128, 313, 24, 186, 250, 13, 263, 158, 180, 127, 368, 75, 214, 304, 184, 152, 329, 237, 134, 322, 197, 310, 321, 44, 355, 147, 171, 324, 380, 241, 77, 92, 309, 96, 157, 170, 190, 72, 335, 108, 320, 353, 76, 349, 189, 11, 10, 337, 249, 379, 364, 38, 333, 345, 334, 344, 125, 133, 116, 23, 84, 372, 89, 15, 175, 266, 341, 94, 58, 53, 248, 50, 296, 365, 293, 151, 210, 51, 257, 291, 378, 202, 109, 196, 194, 381, 246, 70, 3, 79, 121, 87, 114, 282, 176, 277, 217, 30, 331, 137, 143, 115, 240, 360, 18, 311, 225, 57, 346, 169, 200, 155, 204, 99, 201, 80, 359, 135, 328, 278, 276, 188, 43, 66, 318, 28, 239, 149, 283, 244, 350, 164, 100, 262, 7, 8, 251, 46, 382, 48, 62, 148, 111, 339, 105, 193, 88, 131, 140, 195, 362, 9, 232, 95, 101, 234, 305, 370, 352, 4, 319, 377, 106, 63, 323, 168, 97, 27, 269, 223, 252, 233, 264, 315, 93, 90, 376, 224, 120, 112, 356, 265, 17, 371, 332, 104, 14, 138, 71, 270, 357, 117, 287, 49, 212, 74, 369, 33, 253, 383, 271, 86, 245, 123, 52, 60, 255, 209, 375, 35, 39, 325, 374, 40, 267, 326, 288, 178, 215, 31, 358, 16, 12, 275, 191, 373, 185, 367, 6, 65, 132, 327, 222, 228, 183, 289, 55, 156, 285, 167, 272, 47, 279, 124, 59, 220, 227, 136, 205, 187, 307, 82, 231, 81, 351, 182, 2, 45, 216, 281, 229, 32, 292, 294, 159, 1, 145, 173, 163, 54, 366, 150, 103, 107, 36, 110, 259, 98, 130, 25, 174, 218, 242, 254, 162, 261, 126, 280, 21, 260, 64, 22, 268, 301, 20, 219, 299, 199, 146, 0, 172, 160, 41, 198, 181, 258, 230, 161, 113, 342, 336, 273, 208, 5, 102, 338, 69, 314, 343, 317, 221, 68, 243, 83, 165, 308, 122, 247, 206, 34, 274, 119, 361, 347, 290, 42, 154, 256, 78, 91, 238, 153, 312, 330, 29, 297, 139, 166, 141, 306, 211, 192, 67, 26, 19, 203, 235, 298, 73, 348] + 6: [97, 71, 323, 331, 40, 90, 161, 164, 247, 257, 342, 373, 142, 363, 25, 33, 52, 186, 204, 275, 315, 35, 320, 325, 308, 218, 353, 265, 153, 252, 75, 132, 149, 135, 167, 214, 322, 248, 299, 74, 51, 192, 241, 273, 170, 61, 72, 65, 76, 23, 333, 134, 360, 374, 19, 93, 382, 20, 2, 314, 274, 133, 157, 243, 346, 123, 17, 38, 127, 340, 191, 228, 376, 231, 327, 206, 372, 96, 235, 21, 140, 141, 344, 200, 32, 109, 158, 220, 225, 168, 125, 226, 201, 91, 1, 172, 305, 319, 301, 101, 332, 104, 126, 215, 357, 47, 171, 227, 53, 318, 208, 270, 12, 211, 207, 148, 16, 312, 131, 78, 343, 163, 287, 113, 68, 15, 237, 122, 124, 347, 102, 213, 174, 45, 255, 259, 236, 77, 271, 217, 42, 244, 89, 238, 258, 350, 112, 234, 49, 383, 262, 355, 268, 39, 121, 7, 253, 173, 119, 239, 245, 221, 6, 82, 3, 73, 117, 26, 240, 175, 56, 370, 380, 139, 280, 330, 193, 310, 256, 29, 281, 9, 368, 156, 41, 169, 365, 60, 105, 366, 266, 92, 99, 371, 293, 129, 377, 282, 58, 197, 199, 364, 85, 302, 66, 67, 100, 289, 352, 321, 154, 335, 83, 379, 290, 339, 233, 31, 232, 155, 115, 202, 98, 159, 81, 54, 328, 250, 166, 120, 367, 184, 369, 111, 296, 284, 48, 309, 337, 263, 177, 205, 88, 178, 128, 190, 286, 63, 18, 272, 251, 356, 146, 326, 187, 79, 185, 37, 10, 110, 14, 198, 288, 195, 297, 180, 94, 291, 179, 279, 136, 276, 138, 358, 108, 162, 212, 209, 181, 34, 354, 30, 219, 116, 261, 381, 285, 183, 59, 46, 216, 118, 165, 223, 143, 137, 145, 50, 62, 303, 295, 11, 13, 196, 176, 304, 264, 242, 316, 147, 203, 294, 5, 229, 307, 188, 277, 317, 324, 43, 329, 224, 106, 338, 103, 341, 359, 246, 345, 230, 64, 69, 362, 70, 292, 222, 160, 55, 260, 84, 300, 311, 107, 361, 194, 8, 152, 313, 27, 28, 375, 57, 0, 278, 144, 87, 254, 334, 80, 24, 249, 269, 378, 189, 151, 336, 114, 22, 4, 349, 150, 348, 306, 267, 36, 283, 95, 182, 44, 210, 86, 298, 130, 351] + 7: [327, 321, 19, 56, 120, 10, 228, 156, 135, 171, 50, 60, 7, 2, 372, 380, 323, 320, 112, 304, 138, 257, 338, 364, 274, 337, 158, 92, 152, 22, 363, 334, 223, 371, 357, 42, 49, 183, 84, 213, 204, 305, 68, 209, 43, 147, 179, 91, 292, 14, 23, 224, 140, 6, 146, 216, 231, 242, 127, 202, 175, 281, 168, 221, 264, 200, 161, 203, 59, 78, 355, 110, 182, 329, 267, 235, 214, 137, 196, 351, 85, 128, 34, 348, 275, 48, 313, 288, 330, 185, 141, 149, 287, 297, 205, 276, 192, 378, 249, 270, 90, 358, 51, 116, 286, 252, 172, 365, 263, 118, 11, 31, 377, 245, 44, 261, 106, 39, 66, 290, 64, 99, 324, 55, 62, 109, 86, 291, 166, 379, 150, 145, 41, 71, 73, 285, 219, 26, 5, 251, 366, 88, 315, 376, 375, 253, 104, 74, 383, 368, 237, 306, 37, 295, 246, 143, 142, 13, 302, 307, 284, 167, 107, 8, 101, 367, 159, 54, 229, 139, 181, 217, 208, 241, 211, 227, 93, 96, 266, 144, 265, 280, 256, 125, 296, 314, 186, 310, 79, 72, 195, 199, 343, 271, 27, 248, 325, 16, 32, 134, 184, 218, 198, 282, 382, 342, 94, 100, 356, 206, 178, 309, 153, 123, 381, 240, 340, 46, 174, 4, 210, 98, 273, 233, 115, 36, 45, 12, 105, 28, 77, 119, 170, 191, 335, 373, 126, 69, 230, 165, 18, 187, 244, 17, 255, 243, 293, 40, 193, 80, 317, 160, 103, 176, 328, 349, 21, 300, 102, 301, 269, 194, 332, 197, 201, 260, 0, 29, 177, 344, 58, 353, 35, 33, 299, 122, 25, 350, 114, 278, 319, 234, 220, 247, 38, 254, 124, 279, 362, 133, 370, 262, 117, 259, 190, 232, 225, 111, 3, 236, 298, 239, 61, 289, 70, 333, 83, 283, 318, 316, 359, 76, 369, 361, 131, 82, 374, 24, 180, 155, 222, 169, 346, 331, 163, 108, 212, 215, 81, 132, 322, 65, 87, 312, 303, 326, 341, 308, 188, 164, 339, 311, 136, 67, 268, 207, 57, 352, 162, 336, 15, 113, 250, 189, 130, 226, 151, 75, 95, 347, 157, 258, 354, 294, 360, 89, 20, 121, 345, 173, 277, 30, 272, 238, 97, 47, 148, 9, 129, 53, 52, 1, 63, 154] + 8: [360, 32, 378, 111, 160, 0, 227, 102, 58, 170, 234, 207, 253, 303, 125, 28, 190, 236, 193, 355, 19, 262, 132, 96, 290, 333, 304, 332, 87, 307, 260, 324, 51, 63, 213, 100, 359, 115, 34, 368, 2, 71, 327, 134, 370, 210, 163, 1, 5, 157, 346, 92, 50, 4, 293, 16, 369, 180, 331, 376, 67, 337, 97, 109, 278, 341, 174, 382, 145, 220, 363, 296, 31, 228, 168, 23, 215, 318, 98, 37, 159, 146, 305, 55, 135, 64, 377, 133, 219, 197, 118, 41, 205, 76, 343, 127, 21, 282, 364, 27, 40, 211, 230, 47, 35, 351, 70, 242, 367, 154, 11, 150, 178, 177, 212, 299, 89, 24, 286, 176, 264, 88, 18, 75, 216, 48, 206, 149, 79, 365, 129, 342, 239, 381, 209, 69, 169, 120, 335, 259, 200, 287, 348, 361, 201, 272, 349, 172, 121, 164, 263, 310, 265, 357, 103, 288, 362, 243, 44, 188, 65, 141, 217, 184, 85, 33, 15, 345, 162, 241, 328, 356, 275, 233, 117, 352, 235, 153, 84, 289, 321, 336, 294, 83, 165, 192, 254, 110, 325, 277, 171, 49, 10, 173, 316, 22, 17, 334, 383, 373, 339, 326, 271, 57, 62, 312, 380, 366, 330, 113, 114, 350, 155, 185, 77, 94, 221, 12, 255, 107, 261, 317, 358, 25, 99, 295, 268, 203, 308, 13, 95, 181, 78, 112, 42, 73, 56, 9, 245, 142, 353, 224, 186, 225, 179, 189, 101, 237, 144, 175, 285, 80, 152, 108, 223, 311, 202, 130, 52, 151, 136, 347, 208, 139, 371, 7, 251, 128, 298, 196, 60, 90, 266, 46, 158, 274, 229, 161, 20, 280, 147, 300, 66, 249, 372, 54, 244, 344, 204, 106, 252, 86, 6, 198, 45, 116, 140, 375, 354, 138, 182, 302, 30, 131, 38, 309, 59, 248, 199, 14, 269, 301, 281, 338, 291, 148, 283, 167, 8, 191, 313, 315, 270, 82, 273, 238, 276, 166, 240, 267, 91, 379, 105, 306, 123, 43, 104, 292, 36, 156, 226, 137, 187, 232, 314, 323, 340, 124, 214, 218, 122, 195, 231, 320, 72, 297, 126, 29, 319, 222, 250, 246, 81, 39, 119, 183, 61, 247, 68, 93, 329, 256, 3, 279, 74, 374, 258, 194, 284, 257, 143, 322, 26, 53] + 9: [92, 320, 347, 368, 40, 164, 236, 283, 338, 365, 234, 57, 304, 245, 253, 227, 44, 182, 152, 168, 340, 348, 173, 26, 255, 187, 321, 150, 276, 112, 76, 191, 257, 157, 27, 369, 349, 144, 6, 139, 343, 117, 301, 383, 362, 250, 53, 24, 327, 121, 360, 180, 267, 104, 60, 84, 199, 284, 61, 77, 355, 51, 161, 54, 105, 5, 379, 110, 178, 345, 55, 126, 31, 198, 14, 153, 97, 72, 235, 62, 224, 290, 85, 122, 226, 344, 23, 88, 142, 336, 80, 203, 298, 293, 204, 265, 94, 37, 136, 363, 107, 171, 354, 373, 89, 188, 378, 25, 263, 29, 67, 158, 247, 206, 8, 46, 310, 326, 200, 7, 295, 381, 209, 30, 231, 4, 202, 382, 286, 335, 273, 156, 47, 351, 341, 82, 318, 323, 359, 100, 371, 346, 176, 28, 185, 184, 271, 317, 0, 256, 237, 303, 42, 48, 240, 91, 313, 179, 214, 285, 334, 195, 106, 52, 213, 333, 160, 249, 316, 155, 281, 151, 306, 269, 288, 307, 300, 229, 59, 361, 90, 222, 210, 367, 228, 186, 16, 319, 291, 270, 193, 63, 262, 165, 329, 133, 297, 145, 230, 246, 261, 125, 374, 194, 49, 74, 287, 312, 377, 163, 123, 75, 322, 216, 34, 366, 64, 219, 370, 9, 135, 259, 11, 208, 254, 38, 170, 119, 244, 350, 159, 98, 120, 128, 141, 50, 205, 375, 325, 73, 95, 353, 380, 258, 302, 20, 124, 339, 2, 251, 243, 113, 103, 299, 242, 131, 154, 364, 274, 65, 275, 305, 68, 101, 147, 357, 138, 22, 3, 192, 111, 248, 174, 232, 167, 223, 10, 252, 217, 177, 292, 99, 36, 308, 114, 330, 140, 296, 212, 172, 376, 132, 137, 183, 238, 130, 45, 12, 43, 266, 78, 19, 272, 239, 33, 311, 218, 18, 233, 162, 337, 93, 166, 215, 201, 342, 169, 87, 189, 109, 148, 71, 108, 118, 81, 260, 83, 309, 289, 41, 181, 17, 79, 352, 225, 279, 143, 280, 1, 56, 277, 35, 207, 39, 190, 21, 241, 149, 331, 372, 15, 102, 356, 116, 66, 324, 115, 134, 58, 294, 220, 196, 146, 32, 264, 129, 70, 13, 358, 211, 86, 328, 96, 315, 127, 282, 268, 314, 332, 175, 197, 221, 69, 278] + 10: [319, 349, 290, 295, 358, 366, 29, 68, 283, 336, 297, 369, 5, 160, 110, 70, 236, 253, 166, 176, 23, 82, 291, 324, 300, 227, 10, 14, 242, 28, 43, 188, 228, 89, 238, 381, 320, 27, 383, 313, 159, 4, 333, 63, 277, 260, 52, 270, 365, 308, 348, 382, 72, 338, 278, 220, 62, 53, 352, 202, 194, 66, 312, 98, 229, 85, 51, 317, 131, 217, 44, 150, 264, 138, 293, 190, 90, 106, 97, 323, 326, 224, 157, 152, 55, 19, 353, 77, 198, 122, 230, 275, 169, 17, 281, 156, 180, 153, 115, 204, 132, 296, 41, 174, 244, 187, 158, 191, 151, 263, 213, 342, 199, 288, 328, 193, 257, 206, 200, 361, 265, 171, 380, 377, 155, 311, 74, 254, 307, 305, 6, 347, 24, 215, 201, 136, 231, 21, 112, 36, 343, 80, 88, 30, 121, 73, 177, 240, 306, 232, 378, 340, 282, 139, 94, 84, 216, 102, 182, 11, 38, 103, 314, 162, 185, 287, 273, 143, 18, 79, 210, 243, 189, 0, 60, 267, 50, 350, 168, 344, 286, 101, 241, 272, 360, 33, 142, 371, 239, 96, 292, 207, 31, 57, 40, 76, 208, 145, 219, 25, 140, 209, 303, 83, 164, 225, 351, 67, 148, 128, 81, 379, 45, 364, 205, 172, 146, 92, 114, 7, 316, 250, 75, 37, 332, 299, 147, 104, 310, 2, 78, 91, 49, 32, 129, 339, 39, 362, 119, 35, 301, 309, 117, 58, 186, 376, 56, 9, 247, 298, 375, 120, 294, 331, 161, 47, 252, 271, 3, 197, 12, 95, 355, 16, 192, 367, 179, 196, 284, 359, 107, 335, 64, 123, 373, 133, 255, 126, 178, 124, 363, 266, 214, 256, 125, 163, 86, 322, 246, 167, 134, 237, 374, 218, 135, 20, 15, 65, 173, 276, 93, 69, 144, 269, 368, 212, 105, 26, 71, 226, 111, 34, 268, 279, 8, 211, 259, 223, 46, 357, 318, 183, 235, 203, 137, 337, 22, 248, 130, 280, 87, 329, 184, 261, 222, 245, 181, 325, 1, 330, 345, 61, 116, 346, 42, 370, 372, 251, 108, 289, 170, 154, 234, 249, 341, 59, 258, 113, 127, 165, 48, 262, 195, 304, 356, 149, 233, 327, 109, 175, 100, 118, 221, 141, 54, 99, 274, 285, 321, 315, 354, 13, 302, 334] + 11: [130, 373, 289, 279, 383, 184, 132, 53, 256, 243, 335, 269, 314, 320, 226, 365, 292, 57, 382, 214, 181, 273, 42, 241, 296, 250, 45, 151, 294, 125, 113, 310, 180, 333, 32, 12, 163, 207, 280, 10, 336, 120, 129, 140, 272, 72, 257, 105, 110, 128, 297, 27, 55, 366, 64, 271, 380, 137, 50, 9, 98, 52, 303, 80, 255, 362, 264, 43, 21, 114, 206, 109, 377, 165, 196, 117, 89, 254, 353, 19, 103, 344, 253, 261, 339, 178, 148, 283, 160, 301, 234, 142, 288, 183, 76, 308, 87, 29, 95, 350, 197, 208, 291, 48, 322, 68, 155, 61, 90, 162, 39, 47, 195, 259, 338, 143, 200, 361, 111, 191, 83, 102, 73, 313, 93, 96, 265, 237, 375, 139, 317, 134, 215, 267, 131, 6, 92, 186, 312, 8, 274, 352, 372, 307, 85, 199, 343, 78, 2, 224, 81, 349, 230, 5, 60, 202, 150, 157, 330, 23, 75, 14, 97, 91, 263, 287, 167, 212, 235, 325, 284, 136, 346, 251, 211, 302, 355, 104, 290, 329, 221, 300, 141, 169, 124, 86, 242, 311, 182, 173, 268, 334, 121, 190, 217, 17, 94, 172, 30, 354, 194, 37, 175, 28, 11, 145, 345, 177, 170, 374, 59, 359, 231, 108, 216, 16, 36, 233, 0, 154, 319, 315, 168, 364, 358, 106, 277, 153, 116, 171, 22, 228, 367, 340, 35, 378, 46, 360, 369, 189, 44, 99, 332, 20, 371, 341, 327, 34, 7, 112, 328, 232, 248, 38, 351, 244, 159, 298, 31, 324, 18, 306, 379, 220, 100, 326, 152, 204, 127, 49, 101, 135, 188, 356, 1, 239, 126, 119, 247, 70, 69, 26, 147, 293, 258, 185, 304, 223, 357, 149, 144, 282, 348, 203, 209, 118, 174, 240, 305, 281, 285, 65, 218, 13, 66, 347, 245, 158, 138, 323, 229, 276, 15, 40, 24, 205, 161, 176, 295, 115, 79, 4, 278, 321, 156, 3, 192, 381, 107, 316, 270, 164, 225, 201, 71, 82, 252, 376, 54, 286, 249, 56, 63, 187, 62, 210, 238, 122, 77, 227, 236, 146, 213, 133, 41, 179, 309, 84, 342, 25, 260, 363, 331, 337, 262, 166, 219, 67, 33, 222, 368, 123, 299, 88, 275, 318, 370, 198, 58, 193, 74, 266, 246, 51] + 12: [255, 250, 366, 199, 73, 110, 225, 272, 372, 220, 302, 42, 204, 169, 112, 44, 125, 100, 166, 64, 66, 99, 222, 162, 120, 83, 319, 336, 105, 247, 213, 377, 331, 240, 43, 53, 130, 155, 0, 58, 86, 310, 230, 206, 359, 141, 239, 373, 158, 180, 283, 136, 122, 352, 178, 338, 88, 21, 56, 353, 327, 266, 362, 301, 315, 187, 12, 284, 117, 79, 275, 95, 149, 281, 89, 273, 15, 165, 335, 5, 251, 258, 92, 324, 78, 195, 323, 322, 14, 185, 45, 65, 329, 237, 143, 311, 226, 98, 205, 371, 294, 200, 298, 257, 246, 212, 118, 248, 190, 50, 278, 16, 271, 286, 234, 321, 32, 252, 320, 132, 303, 233, 6, 60, 107, 144, 114, 325, 293, 146, 343, 221, 127, 192, 339, 108, 57, 91, 41, 173, 183, 383, 55, 182, 223, 290, 295, 68, 102, 241, 231, 189, 379, 297, 81, 210, 82, 97, 123, 268, 33, 326, 350, 67, 270, 279, 96, 307, 177, 103, 134, 382, 264, 63, 253, 161, 235, 106, 47, 202, 378, 287, 259, 229, 265, 291, 304, 232, 355, 174, 375, 228, 69, 71, 176, 317, 116, 129, 1, 224, 140, 179, 72, 367, 243, 126, 354, 25, 10, 318, 9, 119, 334, 244, 211, 19, 150, 296, 3, 152, 133, 157, 254, 61, 351, 374, 113, 7, 172, 289, 75, 194, 27, 59, 357, 280, 115, 24, 282, 314, 333, 349, 340, 216, 300, 164, 376, 4, 76, 285, 34, 242, 203, 124, 208, 215, 39, 380, 142, 256, 36, 312, 54, 111, 46, 22, 260, 330, 153, 238, 193, 249, 214, 70, 30, 360, 341, 292, 198, 346, 101, 308, 191, 84, 305, 128, 18, 184, 156, 267, 181, 137, 104, 135, 358, 37, 40, 207, 347, 26, 363, 109, 151, 147, 277, 23, 227, 313, 13, 365, 236, 196, 49, 168, 90, 368, 364, 74, 370, 262, 121, 148, 131, 94, 35, 175, 51, 87, 288, 31, 309, 356, 38, 85, 316, 328, 48, 167, 306, 348, 209, 337, 201, 139, 11, 274, 345, 342, 381, 344, 170, 332, 171, 197, 52, 29, 20, 186, 299, 269, 245, 17, 369, 276, 2, 80, 77, 62, 154, 217, 8, 163, 219, 28, 361, 218, 160, 261, 188, 138, 159, 263, 93, 145] + 13: [170, 350, 140, 155, 230, 316, 78, 192, 32, 204, 7, 175, 108, 320, 62, 139, 379, 308, 103, 157, 278, 66, 324, 315, 232, 158, 179, 333, 36, 244, 217, 60, 90, 120, 206, 126, 18, 91, 79, 166, 68, 50, 358, 203, 152, 160, 301, 33, 195, 64, 266, 240, 256, 288, 268, 213, 346, 258, 218, 314, 87, 111, 143, 339, 183, 48, 296, 124, 83, 349, 367, 27, 80, 109, 297, 42, 225, 125, 59, 13, 343, 293, 200, 216, 44, 162, 16, 299, 25, 52, 357, 53, 190, 212, 156, 165, 362, 370, 364, 208, 144, 29, 326, 311, 119, 28, 47, 193, 329, 338, 75, 334, 141, 243, 153, 187, 353, 63, 354, 247, 114, 17, 176, 309, 375, 374, 223, 245, 38, 1, 264, 277, 14, 340, 323, 273, 65, 196, 241, 325, 19, 226, 99, 82, 149, 110, 304, 289, 300, 145, 76, 56, 123, 249, 290, 302, 328, 100, 306, 31, 373, 239, 222, 319, 70, 138, 132, 255, 23, 342, 171, 69, 6, 26, 164, 345, 360, 233, 236, 322, 260, 95, 46, 188, 122, 344, 380, 220, 154, 182, 173, 351, 312, 55, 298, 112, 2, 228, 15, 136, 94, 214, 352, 148, 280, 276, 335, 369, 178, 382, 146, 106, 71, 104, 215, 368, 189, 231, 142, 184, 363, 321, 129, 161, 113, 115, 101, 181, 238, 201, 199, 49, 292, 43, 327, 337, 133, 72, 117, 224, 12, 135, 88, 227, 67, 219, 253, 286, 250, 270, 284, 347, 30, 21, 186, 198, 281, 355, 151, 248, 348, 73, 252, 251, 274, 172, 11, 58, 229, 237, 177, 267, 263, 377, 336, 371, 318, 51, 366, 45, 294, 211, 93, 89, 305, 61, 205, 269, 9, 282, 310, 295, 261, 168, 283, 105, 92, 5, 185, 272, 194, 41, 134, 159, 54, 22, 361, 331, 257, 130, 4, 317, 137, 24, 102, 307, 202, 98, 121, 163, 174, 313, 383, 20, 77, 169, 254, 118, 275, 10, 291, 37, 259, 150, 167, 332, 57, 265, 209, 279, 40, 210, 74, 303, 127, 235, 372, 97, 81, 39, 330, 116, 356, 207, 107, 221, 381, 234, 180, 191, 3, 86, 197, 131, 246, 378, 359, 8, 84, 287, 85, 128, 96, 0, 271, 35, 376, 147, 262, 34, 242, 365, 285, 341] + 14: [229, 249, 295, 121, 205, 260, 289, 331, 50, 88, 165, 174, 160, 296, 225, 240, 365, 194, 303, 210, 130, 318, 83, 212, 345, 166, 332, 145, 284, 316, 103, 58, 7, 134, 292, 315, 255, 250, 25, 172, 234, 63, 262, 46, 11, 247, 272, 73, 70, 213, 314, 193, 109, 222, 1, 251, 322, 47, 243, 351, 211, 124, 13, 343, 382, 105, 168, 153, 334, 185, 170, 119, 143, 198, 111, 246, 40, 82, 350, 376, 87, 281, 218, 232, 4, 254, 79, 328, 333, 341, 223, 273, 127, 120, 320, 67, 15, 235, 154, 135, 349, 48, 89, 10, 267, 140, 381, 98, 76, 146, 61, 161, 203, 226, 244, 302, 195, 81, 144, 138, 204, 348, 162, 158, 159, 227, 253, 248, 74, 139, 219, 196, 110, 118, 317, 33, 370, 178, 0, 93, 171, 258, 233, 53, 30, 337, 371, 197, 326, 169, 156, 152, 94, 78, 180, 252, 2, 353, 32, 220, 357, 104, 129, 191, 114, 136, 35, 283, 209, 150, 177, 208, 69, 37, 306, 189, 278, 361, 84, 307, 259, 186, 97, 125, 184, 301, 359, 335, 358, 9, 293, 27, 75, 344, 342, 270, 291, 277, 86, 336, 309, 179, 18, 340, 34, 239, 231, 26, 319, 22, 330, 300, 375, 14, 55, 51, 122, 268, 64, 265, 324, 167, 221, 242, 214, 238, 24, 236, 327, 49, 39, 41, 369, 339, 237, 264, 6, 338, 107, 65, 126, 175, 3, 216, 201, 108, 298, 202, 321, 290, 57, 72, 355, 112, 117, 294, 68, 363, 276, 200, 257, 190, 66, 368, 378, 199, 95, 44, 206, 325, 347, 313, 173, 36, 181, 188, 308, 383, 100, 271, 379, 256, 373, 217, 42, 230, 362, 54, 155, 17, 52, 157, 215, 102, 90, 12, 285, 263, 45, 352, 71, 8, 85, 374, 288, 266, 176, 115, 329, 305, 62, 192, 275, 60, 141, 354, 323, 187, 101, 148, 106, 19, 366, 131, 147, 137, 151, 360, 123, 133, 377, 282, 311, 16, 80, 29, 56, 287, 228, 279, 149, 128, 116, 364, 28, 20, 367, 91, 274, 164, 23, 21, 297, 286, 5, 356, 310, 304, 92, 113, 261, 77, 142, 269, 163, 346, 380, 183, 43, 299, 132, 38, 372, 207, 99, 182, 59, 245, 241, 224, 96, 312, 31, 280] + 15: [130, 245, 75, 131, 322, 366, 260, 93, 281, 270, 85, 328, 11, 220, 27, 315, 264, 198, 325, 127, 192, 368, 356, 276, 0, 308, 216, 364, 188, 299, 122, 156, 345, 348, 25, 144, 15, 43, 365, 84, 262, 44, 125, 214, 132, 33, 46, 6, 88, 269, 294, 71, 273, 58, 136, 302, 336, 249, 19, 152, 359, 78, 49, 22, 226, 272, 91, 153, 154, 287, 297, 186, 160, 255, 133, 323, 62, 349, 123, 52, 380, 243, 288, 219, 227, 56, 18, 182, 194, 8, 165, 211, 218, 280, 178, 21, 57, 203, 83, 109, 5, 263, 12, 354, 172, 197, 124, 286, 362, 215, 35, 196, 209, 309, 13, 267, 334, 303, 112, 108, 251, 119, 48, 320, 170, 247, 206, 253, 92, 116, 310, 179, 324, 284, 23, 138, 335, 238, 282, 107, 45, 54, 176, 183, 155, 252, 151, 177, 207, 105, 254, 271, 26, 265, 187, 41, 355, 333, 360, 1, 352, 275, 229, 180, 224, 190, 293, 103, 200, 74, 221, 168, 24, 142, 34, 381, 208, 278, 318, 120, 258, 236, 90, 256, 164, 59, 268, 347, 314, 274, 184, 223, 94, 159, 60, 173, 235, 117, 77, 61, 137, 378, 374, 304, 128, 266, 9, 342, 379, 2, 114, 97, 234, 73, 369, 307, 217, 326, 80, 353, 351, 246, 233, 339, 283, 63, 149, 346, 175, 231, 329, 311, 162, 102, 50, 313, 373, 213, 370, 17, 148, 332, 317, 163, 31, 279, 69, 191, 169, 3, 201, 70, 104, 185, 232, 296, 338, 145, 341, 301, 375, 257, 14, 292, 134, 38, 121, 277, 115, 82, 87, 306, 383, 65, 312, 110, 68, 189, 146, 113, 225, 166, 331, 321, 244, 239, 29, 222, 298, 300, 20, 135, 230, 248, 10, 53, 289, 161, 89, 330, 290, 39, 51, 344, 106, 212, 150, 67, 55, 79, 32, 361, 261, 199, 237, 193, 285, 86, 343, 72, 99, 147, 129, 66, 340, 291, 376, 242, 47, 372, 205, 382, 337, 316, 126, 98, 350, 371, 157, 174, 36, 101, 250, 30, 37, 167, 327, 181, 76, 367, 139, 143, 100, 28, 96, 118, 204, 210, 158, 195, 4, 202, 16, 241, 357, 42, 81, 377, 95, 358, 140, 228, 259, 295, 40, 111, 319, 305, 363, 141, 64, 240, 7, 171] + 16: [134, 366, 368, 1, 98, 344, 3, 148, 156, 249, 257, 353, 15, 345, 13, 69, 215, 239, 320, 94, 111, 126, 247, 309, 286, 278, 296, 325, 376, 361, 362, 10, 53, 133, 161, 191, 120, 365, 66, 41, 106, 25, 190, 107, 332, 299, 192, 72, 275, 310, 37, 195, 124, 132, 244, 145, 149, 6, 197, 179, 100, 212, 307, 380, 301, 375, 93, 62, 170, 79, 263, 204, 140, 146, 71, 56, 45, 104, 223, 363, 110, 352, 327, 222, 369, 189, 85, 21, 70, 160, 206, 48, 315, 251, 164, 291, 152, 171, 199, 248, 229, 22, 57, 30, 81, 150, 349, 326, 234, 35, 236, 321, 155, 36, 290, 313, 194, 14, 279, 136, 262, 374, 235, 177, 139, 184, 225, 336, 91, 88, 359, 173, 92, 205, 46, 52, 281, 305, 188, 358, 109, 127, 50, 351, 7, 5, 213, 314, 103, 34, 137, 214, 193, 157, 77, 12, 43, 348, 295, 306, 58, 219, 9, 211, 142, 166, 176, 112, 165, 259, 364, 84, 80, 76, 297, 242, 196, 373, 241, 23, 147, 318, 141, 285, 316, 167, 322, 60, 201, 128, 116, 40, 269, 203, 187, 27, 82, 220, 317, 340, 272, 17, 245, 74, 64, 342, 121, 89, 119, 8, 276, 95, 304, 113, 61, 33, 44, 289, 287, 294, 99, 356, 96, 54, 207, 284, 217, 130, 303, 143, 383, 65, 292, 232, 4, 200, 266, 324, 256, 135, 118, 90, 265, 108, 175, 227, 339, 178, 319, 131, 138, 101, 280, 129, 209, 181, 73, 182, 198, 228, 83, 261, 260, 254, 75, 49, 283, 346, 377, 357, 370, 158, 105, 20, 210, 169, 271, 144, 298, 180, 38, 379, 238, 102, 240, 47, 216, 329, 11, 273, 255, 16, 335, 31, 174, 18, 378, 268, 230, 382, 162, 86, 24, 226, 51, 231, 19, 208, 55, 218, 243, 28, 68, 338, 302, 367, 237, 159, 42, 354, 267, 381, 117, 32, 67, 277, 224, 270, 328, 153, 202, 221, 333, 172, 168, 63, 330, 312, 183, 87, 59, 282, 334, 252, 337, 186, 258, 355, 125, 300, 274, 246, 115, 0, 122, 264, 253, 360, 26, 288, 331, 154, 250, 347, 97, 39, 372, 29, 323, 163, 308, 341, 185, 114, 78, 123, 2, 311, 293, 151, 343, 233, 371, 350] + 17: [131, 170, 202, 203, 235, 237, 251, 283, 312, 320, 325, 354, 119, 169, 14, 86, 122, 127, 358, 69, 72, 121, 152, 159, 284, 321, 156, 209, 252, 301, 333, 123, 174, 293, 34, 54, 236, 228, 162, 248, 37, 95, 192, 150, 368, 247, 111, 281, 79, 296, 371, 332, 154, 110, 369, 233, 178, 71, 35, 322, 337, 194, 367, 376, 336, 21, 372, 318, 242, 77, 51, 68, 375, 8, 46, 272, 260, 49, 238, 128, 32, 176, 0, 268, 261, 29, 96, 373, 18, 7, 115, 102, 329, 315, 201, 17, 220, 351, 290, 180, 292, 19, 365, 120, 161, 53, 109, 334, 250, 2, 234, 335, 84, 45, 196, 249, 151, 65, 277, 348, 188, 130, 370, 199, 135, 42, 295, 262, 48, 344, 160, 285, 303, 193, 107, 313, 214, 166, 341, 360, 326, 255, 288, 306, 108, 225, 297, 148, 327, 269, 81, 302, 55, 57, 356, 353, 200, 359, 133, 73, 98, 191, 363, 379, 25, 76, 30, 289, 15, 155, 189, 317, 143, 259, 41, 181, 93, 85, 316, 66, 383, 117, 347, 207, 10, 168, 215, 382, 239, 210, 355, 223, 100, 136, 324, 39, 92, 171, 13, 309, 213, 67, 270, 11, 26, 311, 125, 134, 273, 245, 47, 101, 342, 364, 350, 129, 330, 78, 246, 175, 206, 298, 147, 118, 195, 112, 352, 224, 305, 357, 1, 75, 22, 139, 59, 198, 240, 217, 88, 280, 294, 254, 185, 222, 231, 146, 173, 204, 244, 287, 182, 378, 362, 219, 5, 157, 380, 142, 264, 271, 94, 106, 144, 229, 374, 91, 275, 104, 211, 263, 61, 3, 310, 64, 83, 23, 28, 12, 114, 276, 97, 99, 331, 187, 179, 274, 163, 257, 212, 87, 286, 339, 103, 70, 340, 338, 27, 124, 50, 349, 138, 366, 232, 56, 38, 74, 20, 205, 361, 267, 132, 149, 307, 126, 343, 172, 381, 165, 36, 186, 300, 308, 164, 4, 80, 58, 279, 167, 218, 256, 43, 216, 258, 345, 227, 40, 105, 346, 323, 6, 145, 9, 82, 89, 44, 158, 31, 184, 208, 319, 113, 328, 24, 33, 60, 221, 299, 16, 153, 230, 140, 190, 266, 62, 177, 314, 304, 253, 265, 278, 63, 291, 226, 137, 197, 52, 243, 282, 116, 377, 90, 183, 241, 141] + 18: [271, 293, 314, 344, 382, 1, 32, 59, 78, 147, 309, 340, 30, 52, 74, 80, 97, 120, 156, 183, 222, 227, 242, 268, 251, 288, 54, 94, 163, 95, 117, 295, 55, 226, 327, 374, 208, 165, 283, 7, 176, 150, 112, 311, 107, 160, 310, 91, 47, 35, 318, 296, 313, 363, 179, 219, 76, 173, 194, 49, 10, 144, 181, 343, 203, 276, 256, 158, 85, 280, 198, 186, 43, 282, 89, 149, 350, 178, 270, 197, 258, 212, 323, 128, 69, 376, 326, 366, 223, 195, 12, 275, 247, 218, 206, 221, 320, 26, 33, 202, 22, 361, 187, 322, 133, 372, 277, 196, 217, 189, 246, 346, 233, 111, 81, 348, 220, 90, 281, 353, 257, 236, 378, 46, 263, 102, 63, 108, 27, 42, 294, 379, 300, 142, 151, 274, 83, 122, 125, 292, 358, 66, 298, 252, 245, 116, 253, 19, 331, 159, 205, 329, 383, 4, 375, 336, 65, 239, 297, 164, 359, 232, 279, 351, 175, 56, 364, 62, 57, 174, 291, 70, 269, 129, 157, 132, 303, 51, 381, 134, 138, 192, 237, 190, 23, 241, 50, 93, 60, 200, 352, 290, 71, 169, 284, 330, 307, 0, 201, 13, 153, 369, 84, 377, 228, 380, 306, 266, 177, 87, 349, 244, 48, 332, 29, 14, 106, 110, 360, 240, 171, 225, 39, 145, 127, 216, 115, 24, 185, 368, 373, 188, 20, 316, 152, 182, 213, 338, 161, 345, 260, 304, 31, 341, 73, 131, 235, 40, 148, 17, 86, 5, 6, 36, 25, 101, 339, 109, 61, 96, 234, 64, 172, 100, 118, 114, 167, 204, 41, 79, 308, 38, 243, 209, 354, 230, 356, 119, 146, 124, 191, 261, 140, 319, 143, 82, 238, 249, 272, 215, 136, 285, 104, 262, 15, 264, 287, 123, 2, 21, 193, 347, 355, 105, 342, 278, 231, 155, 324, 229, 370, 305, 45, 113, 321, 77, 121, 334, 8, 362, 168, 170, 299, 11, 53, 28, 68, 3, 365, 135, 302, 255, 325, 317, 37, 184, 333, 154, 301, 67, 141, 367, 328, 265, 214, 224, 44, 273, 16, 207, 34, 103, 337, 88, 289, 335, 166, 126, 180, 199, 210, 315, 139, 98, 371, 162, 58, 9, 254, 130, 312, 72, 211, 286, 357, 99, 248, 137, 250, 267, 92, 259, 75, 18] + 19: [24, 133, 198, 245, 253, 254, 263, 268, 319, 355, 369, 375, 187, 109, 229, 305, 317, 350, 12, 34, 50, 81, 98, 107, 190, 69, 199, 327, 55, 104, 145, 196, 274, 316, 378, 93, 366, 360, 17, 43, 72, 220, 382, 78, 142, 175, 250, 276, 7, 96, 259, 108, 3, 232, 59, 304, 261, 379, 226, 282, 90, 122, 353, 277, 380, 26, 258, 73, 367, 308, 102, 169, 288, 217, 246, 170, 95, 284, 383, 137, 348, 21, 266, 151, 256, 66, 171, 134, 272, 143, 85, 281, 370, 294, 51, 0, 265, 216, 117, 54, 76, 260, 160, 376, 14, 314, 100, 5, 57, 114, 89, 30, 68, 40, 293, 88, 139, 357, 136, 56, 377, 197, 97, 335, 239, 273, 23, 92, 132, 324, 155, 236, 120, 300, 77, 46, 6, 204, 60, 62, 181, 351, 205, 248, 338, 8, 271, 25, 228, 138, 287, 336, 242, 341, 124, 267, 330, 219, 227, 147, 359, 257, 342, 45, 280, 99, 247, 299, 313, 192, 296, 41, 222, 156, 31, 18, 298, 212, 106, 269, 180, 318, 334, 79, 71, 172, 2, 75, 65, 105, 28, 289, 42, 279, 303, 354, 36, 127, 306, 333, 146, 209, 118, 123, 207, 183, 27, 264, 1, 182, 157, 356, 200, 22, 315, 344, 312, 126, 193, 153, 285, 121, 221, 202, 326, 32, 361, 86, 101, 322, 349, 159, 238, 195, 203, 19, 129, 208, 63, 162, 240, 291, 38, 301, 13, 83, 297, 262, 329, 168, 103, 178, 64, 161, 218, 35, 116, 165, 347, 331, 191, 234, 94, 185, 307, 188, 33, 152, 135, 283, 84, 325, 362, 320, 112, 201, 115, 255, 20, 275, 233, 167, 215, 177, 309, 332, 149, 290, 141, 91, 164, 230, 15, 29, 67, 10, 148, 340, 286, 358, 243, 70, 363, 58, 278, 173, 144, 80, 372, 174, 235, 39, 211, 52, 154, 189, 53, 251, 345, 131, 47, 343, 150, 214, 339, 241, 184, 158, 310, 368, 374, 213, 140, 87, 371, 163, 111, 364, 328, 9, 352, 270, 249, 125, 119, 311, 381, 223, 179, 61, 373, 365, 166, 16, 244, 194, 321, 82, 231, 224, 206, 302, 113, 292, 295, 49, 110, 346, 130, 176, 11, 210, 48, 74, 323, 44, 128, 337, 4, 237, 225, 186, 37, 252] + 20: [161, 200, 206, 214, 262, 263, 266, 281, 299, 303, 331, 350, 116, 52, 54, 61, 76, 107, 137, 167, 168, 179, 181, 189, 334, 17, 117, 198, 241, 242, 249, 260, 277, 280, 27, 51, 98, 284, 233, 97, 170, 56, 356, 329, 31, 155, 143, 269, 153, 65, 58, 279, 304, 360, 283, 225, 192, 298, 197, 288, 278, 125, 184, 235, 150, 246, 3, 80, 333, 152, 203, 171, 302, 290, 101, 194, 367, 345, 39, 75, 21, 323, 220, 1, 336, 273, 49, 240, 213, 274, 338, 159, 120, 341, 251, 38, 82, 10, 160, 13, 26, 60, 232, 374, 296, 354, 23, 237, 144, 209, 342, 176, 375, 166, 310, 230, 319, 47, 234, 44, 66, 158, 259, 190, 35, 346, 236, 24, 366, 64, 261, 69, 193, 62, 95, 92, 267, 215, 5, 253, 88, 163, 103, 244, 99, 32, 128, 320, 218, 247, 294, 201, 139, 205, 129, 255, 87, 204, 221, 347, 348, 309, 169, 257, 210, 67, 293, 109, 43, 105, 141, 132, 364, 146, 42, 252, 89, 11, 19, 373, 369, 363, 229, 258, 177, 53, 22, 377, 217, 124, 271, 186, 45, 353, 111, 292, 57, 328, 86, 33, 202, 151, 63, 383, 268, 224, 228, 286, 90, 41, 196, 29, 254, 321, 355, 222, 145, 195, 91, 140, 239, 378, 165, 78, 231, 156, 157, 68, 4, 130, 324, 216, 6, 175, 238, 15, 351, 28, 7, 275, 335, 291, 164, 135, 74, 219, 131, 370, 316, 40, 85, 301, 46, 307, 93, 362, 104, 314, 300, 365, 191, 352, 207, 108, 282, 379, 148, 264, 121, 368, 178, 332, 212, 59, 305, 306, 8, 339, 162, 381, 126, 0, 372, 380, 337, 142, 83, 183, 122, 226, 325, 9, 315, 25, 119, 106, 312, 199, 276, 245, 37, 112, 185, 227, 357, 118, 256, 20, 94, 289, 16, 133, 174, 2, 371, 187, 182, 30, 208, 73, 326, 79, 72, 295, 272, 376, 330, 311, 71, 343, 340, 127, 327, 115, 136, 70, 14, 287, 188, 50, 172, 359, 18, 12, 84, 285, 313, 147, 317, 123, 138, 382, 322, 265, 34, 248, 308, 113, 102, 344, 55, 110, 318, 96, 77, 36, 114, 297, 154, 250, 211, 149, 81, 173, 270, 180, 358, 361, 243, 100, 223, 349, 48, 134] + 21: [360, 10, 17, 35, 58, 93, 123, 264, 306, 321, 339, 350, 152, 283, 297, 311, 327, 332, 11, 42, 44, 108, 179, 369, 355, 331, 286, 100, 258, 67, 68, 328, 166, 95, 107, 167, 47, 189, 49, 356, 353, 57, 319, 76, 202, 278, 235, 224, 173, 43, 73, 139, 125, 136, 27, 342, 6, 298, 248, 272, 281, 280, 131, 358, 28, 24, 346, 41, 357, 282, 101, 127, 132, 349, 367, 1, 260, 77, 340, 273, 74, 55, 188, 322, 103, 16, 69, 234, 62, 155, 137, 305, 241, 228, 326, 343, 316, 0, 121, 23, 267, 383, 40, 186, 96, 129, 242, 7, 20, 91, 175, 220, 18, 66, 275, 146, 12, 2, 249, 32, 371, 284, 109, 141, 372, 312, 204, 329, 300, 122, 85, 212, 308, 182, 19, 221, 338, 21, 65, 274, 335, 130, 99, 222, 254, 377, 128, 144, 84, 262, 323, 337, 333, 253, 265, 59, 4, 226, 151, 9, 174, 64, 133, 148, 83, 105, 106, 61, 158, 88, 149, 196, 245, 352, 303, 102, 36, 164, 255, 116, 80, 94, 250, 15, 72, 336, 159, 34, 379, 374, 114, 231, 124, 364, 178, 26, 230, 197, 223, 3, 115, 334, 294, 296, 376, 208, 150, 365, 135, 168, 309, 30, 171, 381, 70, 375, 119, 87, 252, 172, 111, 63, 304, 154, 266, 110, 163, 31, 71, 185, 118, 351, 313, 263, 60, 165, 160, 81, 75, 37, 257, 112, 345, 147, 209, 229, 318, 290, 187, 213, 143, 56, 113, 218, 45, 307, 52, 325, 39, 192, 203, 126, 156, 180, 140, 92, 214, 22, 344, 89, 259, 237, 301, 195, 181, 142, 302, 200, 120, 184, 227, 368, 97, 347, 292, 157, 315, 145, 289, 82, 199, 238, 219, 243, 362, 317, 198, 48, 201, 314, 153, 210, 169, 191, 225, 78, 98, 354, 170, 216, 207, 324, 190, 38, 348, 240, 194, 270, 287, 291, 269, 261, 239, 54, 256, 50, 117, 279, 299, 233, 359, 211, 14, 215, 268, 162, 236, 53, 232, 51, 8, 33, 251, 370, 104, 244, 330, 176, 46, 86, 277, 177, 25, 288, 246, 378, 79, 276, 13, 217, 247, 29, 363, 380, 293, 373, 138, 295, 193, 134, 382, 161, 206, 205, 361, 320, 366, 183, 90, 5, 341, 271, 310, 285] + 22: [254, 154, 157, 217, 247, 250, 253, 267, 307, 310, 346, 359, 141, 287, 289, 292, 323, 360, 2, 28, 79, 95, 101, 143, 181, 191, 225, 269, 96, 298, 344, 5, 166, 234, 135, 192, 221, 226, 131, 236, 136, 208, 350, 84, 293, 297, 142, 205, 260, 349, 25, 124, 138, 160, 128, 223, 86, 313, 249, 280, 261, 306, 185, 49, 341, 335, 211, 187, 212, 354, 194, 315, 328, 173, 222, 366, 321, 373, 290, 219, 255, 363, 203, 325, 281, 252, 80, 57, 374, 11, 81, 266, 296, 370, 231, 10, 216, 352, 375, 27, 18, 159, 299, 176, 294, 145, 114, 107, 343, 317, 153, 60, 82, 210, 322, 273, 262, 16, 21, 167, 278, 58, 345, 264, 48, 15, 32, 180, 186, 64, 43, 243, 209, 383, 338, 174, 339, 71, 238, 241, 137, 152, 172, 121, 148, 177, 55, 67, 206, 233, 22, 97, 246, 308, 6, 368, 24, 4, 240, 195, 122, 324, 189, 271, 274, 31, 150, 56, 224, 37, 333, 188, 8, 156, 1, 305, 26, 151, 179, 245, 337, 144, 130, 70, 193, 75, 239, 87, 46, 182, 190, 270, 288, 230, 355, 251, 378, 109, 319, 318, 379, 282, 91, 197, 183, 184, 235, 100, 68, 347, 367, 98, 364, 334, 134, 213, 20, 73, 329, 218, 83, 340, 12, 65, 356, 35, 291, 304, 178, 201, 118, 311, 372, 53, 59, 78, 382, 52, 30, 66, 376, 242, 61, 272, 116, 170, 248, 227, 314, 85, 19, 127, 110, 164, 244, 111, 161, 113, 320, 301, 275, 279, 104, 140, 94, 38, 93, 336, 76, 228, 72, 303, 353, 103, 257, 146, 371, 42, 44, 381, 369, 175, 99, 256, 268, 92, 36, 158, 316, 115, 29, 149, 365, 220, 165, 327, 0, 102, 168, 284, 33, 207, 112, 357, 214, 14, 39, 54, 202, 232, 69, 88, 204, 132, 74, 50, 123, 40, 41, 283, 286, 120, 62, 163, 51, 147, 263, 285, 312, 258, 133, 377, 300, 196, 129, 277, 198, 276, 139, 348, 259, 361, 47, 199, 215, 108, 342, 295, 9, 34, 117, 7, 106, 63, 309, 45, 13, 119, 90, 162, 126, 330, 169, 351, 125, 3, 105, 155, 77, 237, 171, 23, 332, 331, 362, 229, 17, 358, 380, 200, 89, 265, 302, 326] + 23: [26, 43, 235, 286, 320, 37, 81, 242, 265, 64, 168, 288, 314, 375, 357, 22, 262, 13, 234, 291, 39, 75, 189, 8, 124, 221, 98, 352, 108, 355, 277, 30, 142, 18, 3, 15, 172, 233, 210, 65, 45, 21, 373, 31, 141, 60, 226, 4, 275, 70, 255, 125, 54, 68, 104, 34, 42, 198, 366, 216, 71, 309, 224, 93, 179, 335, 25, 188, 381, 112, 251, 113, 305, 56, 102, 40, 17, 196, 206, 227, 88, 318, 166, 29, 126, 59, 111, 152, 254, 307, 107, 285, 158, 299, 332, 110, 328, 372, 213, 313, 160, 321, 346, 349, 61, 343, 217, 296, 264, 182, 190, 271, 361, 333, 339, 243, 248, 377, 199, 186, 175, 84, 130, 249, 215, 53, 76, 134, 267, 230, 208, 380, 100, 103, 159, 35, 38, 298, 250, 200, 336, 253, 364, 293, 132, 327, 57, 131, 228, 1, 204, 0, 115, 367, 118, 145, 187, 279, 169, 209, 317, 157, 207, 344, 268, 161, 165, 105, 167, 324, 127, 133, 149, 222, 178, 109, 155, 205, 238, 89, 244, 197, 241, 92, 191, 356, 5, 58, 354, 79, 322, 278, 323, 12, 11, 340, 229, 101, 301, 73, 121, 214, 337, 83, 252, 203, 304, 283, 91, 44, 137, 143, 14, 260, 350, 183, 85, 269, 374, 263, 151, 359, 325, 363, 122, 308, 383, 246, 72, 87, 192, 41, 220, 292, 273, 258, 114, 369, 266, 289, 300, 306, 67, 371, 312, 129, 231, 90, 212, 106, 62, 63, 135, 240, 180, 78, 36, 311, 52, 348, 163, 33, 276, 345, 177, 24, 94, 257, 378, 136, 66, 303, 7, 139, 368, 117, 281, 173, 232, 148, 74, 272, 329, 347, 16, 297, 46, 116, 225, 82, 156, 315, 219, 290, 365, 162, 245, 97, 202, 138, 51, 331, 176, 6, 146, 10, 370, 193, 128, 123, 140, 247, 69, 201, 261, 270, 379, 342, 153, 330, 28, 99, 9, 334, 274, 295, 27, 218, 50, 195, 185, 341, 164, 294, 319, 259, 120, 48, 154, 55, 184, 95, 119, 223, 302, 236, 310, 20, 376, 77, 358, 86, 326, 237, 96, 382, 174, 362, 353, 171, 211, 19, 47, 351, 80, 194, 32, 284, 280, 239, 282, 181, 147, 170, 287, 2, 144, 23, 150, 316, 256, 338, 49, 360] + 24: [154, 74, 81, 115, 166, 198, 241, 264, 298, 314, 353, 379, 265, 21, 127, 129, 133, 158, 179, 232, 251, 327, 368, 383, 252, 100, 152, 219, 358, 2, 45, 89, 34, 36, 155, 374, 9, 197, 365, 107, 359, 98, 300, 340, 117, 261, 330, 40, 246, 233, 108, 303, 8, 109, 332, 347, 208, 305, 364, 82, 20, 206, 278, 60, 14, 281, 132, 75, 285, 371, 322, 17, 7, 130, 48, 323, 30, 176, 373, 183, 124, 289, 33, 214, 223, 366, 49, 287, 295, 181, 88, 42, 248, 141, 316, 69, 0, 35, 142, 307, 235, 280, 170, 120, 342, 125, 165, 123, 52, 31, 299, 149, 178, 222, 275, 188, 55, 201, 167, 243, 92, 53, 306, 145, 58, 116, 282, 175, 171, 229, 199, 349, 6, 283, 54, 377, 77, 269, 225, 79, 126, 262, 187, 90, 22, 237, 190, 46, 12, 84, 339, 72, 212, 329, 247, 119, 372, 62, 164, 254, 380, 140, 313, 4, 150, 44, 25, 200, 263, 266, 26, 71, 177, 63, 56, 134, 249, 50, 101, 227, 357, 5, 137, 61, 350, 272, 13, 194, 348, 320, 102, 180, 70, 64, 231, 91, 356, 310, 16, 65, 276, 352, 209, 362, 369, 286, 343, 15, 113, 205, 168, 239, 23, 284, 217, 18, 85, 112, 319, 382, 138, 3, 153, 312, 271, 32, 57, 240, 189, 121, 230, 144, 156, 118, 321, 304, 333, 76, 331, 250, 128, 27, 207, 344, 355, 338, 83, 268, 103, 94, 148, 228, 290, 324, 375, 195, 260, 336, 257, 302, 309, 151, 67, 162, 97, 361, 186, 192, 308, 351, 169, 66, 215, 99, 191, 174, 87, 105, 370, 277, 106, 293, 59, 315, 203, 253, 238, 68, 211, 28, 146, 354, 226, 341, 376, 1, 346, 161, 19, 378, 110, 381, 29, 135, 131, 96, 147, 242, 267, 296, 317, 184, 111, 360, 255, 24, 318, 73, 326, 292, 270, 136, 160, 43, 95, 157, 301, 204, 274, 182, 221, 47, 37, 10, 193, 236, 328, 139, 334, 159, 41, 224, 279, 51, 218, 196, 288, 256, 172, 185, 294, 202, 273, 245, 297, 244, 258, 363, 291, 367, 259, 337, 104, 78, 93, 345, 114, 11, 335, 173, 234, 80, 210, 122, 220, 213, 325, 39, 216, 311, 38, 86, 163, 143] + 25: [315, 239, 156, 220, 263, 267, 277, 123, 178, 91, 158, 326, 269, 338, 43, 44, 303, 382, 19, 121, 165, 129, 189, 233, 379, 329, 6, 206, 106, 194, 13, 170, 151, 195, 270, 255, 282, 167, 141, 374, 108, 272, 288, 47, 148, 50, 182, 241, 72, 164, 132, 290, 370, 117, 70, 119, 71, 169, 198, 266, 139, 294, 256, 114, 350, 69, 67, 36, 283, 264, 12, 104, 228, 120, 153, 242, 295, 133, 79, 337, 175, 142, 285, 219, 247, 59, 84, 146, 352, 185, 24, 111, 16, 286, 42, 301, 317, 32, 234, 126, 347, 203, 149, 150, 93, 45, 155, 88, 180, 33, 215, 138, 143, 81, 201, 238, 186, 293, 224, 217, 276, 49, 262, 240, 161, 204, 213, 17, 271, 40, 92, 90, 23, 73, 318, 373, 227, 336, 184, 355, 275, 362, 118, 26, 372, 10, 216, 197, 343, 369, 357, 310, 243, 250, 64, 335, 107, 171, 380, 9, 279, 115, 199, 257, 300, 109, 311, 57, 214, 112, 314, 306, 60, 211, 237, 181, 212, 34, 183, 363, 29, 77, 102, 252, 366, 291, 319, 274, 356, 157, 320, 172, 284, 376, 296, 222, 367, 225, 316, 14, 105, 86, 124, 147, 192, 246, 38, 131, 244, 20, 304, 144, 11, 281, 210, 265, 62, 113, 66, 346, 99, 196, 187, 28, 223, 51, 87, 3, 98, 152, 8, 154, 253, 348, 312, 58, 251, 163, 173, 48, 65, 327, 160, 368, 323, 191, 278, 254, 53, 110, 128, 174, 83, 52, 280, 302, 125, 97, 383, 339, 159, 127, 236, 322, 258, 0, 5, 342, 56, 307, 330, 297, 74, 292, 208, 95, 168, 309, 190, 18, 229, 218, 324, 202, 249, 22, 226, 63, 268, 25, 145, 136, 231, 135, 259, 15, 179, 166, 245, 41, 298, 2, 232, 378, 89, 103, 55, 328, 193, 260, 371, 205, 334, 39, 333, 381, 209, 176, 130, 134, 340, 177, 364, 344, 82, 61, 235, 100, 341, 140, 221, 94, 31, 325, 96, 188, 299, 332, 273, 230, 54, 4, 68, 116, 200, 35, 122, 360, 85, 162, 313, 287, 377, 359, 345, 365, 361, 75, 21, 1, 101, 78, 7, 207, 80, 261, 308, 37, 358, 353, 289, 349, 30, 375, 305, 351, 76, 321, 354, 331, 27, 248, 46, 137] + 26: [136, 218, 228, 234, 242, 246, 261, 305, 348, 351, 374, 375, 237, 54, 68, 86, 96, 106, 111, 139, 169, 170, 175, 209, 89, 78, 346, 56, 250, 95, 207, 60, 203, 260, 299, 1, 269, 318, 370, 134, 317, 29, 208, 23, 155, 210, 132, 213, 217, 178, 251, 215, 50, 300, 361, 119, 282, 248, 273, 168, 172, 156, 151, 118, 252, 93, 352, 39, 163, 283, 286, 288, 90, 55, 166, 122, 35, 144, 362, 100, 193, 295, 322, 309, 80, 62, 360, 186, 239, 231, 3, 9, 314, 381, 366, 52, 6, 376, 128, 2, 20, 45, 177, 364, 244, 206, 330, 379, 292, 22, 180, 174, 66, 98, 75, 340, 34, 101, 85, 84, 380, 355, 200, 190, 24, 81, 298, 336, 335, 284, 211, 310, 11, 293, 131, 191, 79, 159, 256, 12, 275, 216, 222, 58, 124, 266, 167, 259, 97, 17, 41, 114, 47, 297, 296, 345, 18, 316, 30, 204, 27, 51, 115, 123, 121, 59, 199, 28, 53, 344, 315, 359, 99, 268, 171, 129, 165, 74, 205, 46, 236, 337, 135, 67, 287, 194, 0, 143, 140, 77, 274, 147, 21, 279, 158, 265, 353, 294, 107, 153, 105, 40, 258, 164, 181, 16, 238, 110, 179, 173, 195, 350, 125, 72, 270, 188, 291, 289, 227, 328, 323, 241, 277, 37, 197, 333, 240, 235, 253, 116, 369, 332, 319, 103, 65, 196, 49, 112, 326, 257, 339, 120, 307, 382, 26, 102, 357, 32, 189, 141, 154, 243, 230, 224, 278, 146, 138, 327, 15, 267, 232, 325, 334, 313, 150, 254, 341, 126, 198, 306, 127, 202, 182, 220, 303, 13, 320, 160, 38, 92, 176, 44, 285, 290, 372, 7, 347, 377, 5, 149, 308, 36, 48, 229, 57, 378, 312, 264, 8, 324, 233, 162, 130, 108, 311, 142, 192, 161, 117, 368, 43, 358, 281, 304, 137, 245, 272, 152, 104, 212, 383, 329, 64, 373, 349, 338, 183, 88, 31, 367, 365, 73, 363, 157, 185, 4, 271, 70, 33, 249, 148, 356, 71, 87, 25, 301, 255, 14, 214, 263, 94, 223, 82, 262, 201, 113, 280, 219, 354, 145, 63, 371, 276, 226, 247, 133, 343, 76, 187, 225, 331, 91, 342, 184, 69, 61, 83, 321, 221, 10, 19, 302, 109, 42] + 27: [103, 137, 146, 163, 176, 220, 240, 244, 262, 310, 338, 342, 225, 232, 241, 218, 249, 367, 313, 64, 56, 212, 270, 53, 286, 216, 118, 10, 18, 365, 96, 150, 194, 235, 285, 60, 12, 99, 356, 54, 46, 321, 144, 50, 86, 245, 236, 39, 312, 228, 325, 254, 214, 43, 256, 302, 372, 209, 17, 63, 90, 182, 32, 42, 136, 307, 0, 231, 52, 243, 36, 93, 234, 112, 196, 109, 329, 362, 47, 306, 185, 198, 19, 180, 37, 97, 149, 25, 126, 287, 120, 123, 223, 94, 138, 200, 177, 207, 242, 297, 91, 375, 199, 153, 8, 68, 255, 203, 4, 358, 251, 78, 305, 296, 29, 13, 281, 173, 378, 51, 263, 49, 183, 343, 370, 6, 106, 151, 22, 158, 210, 100, 61, 119, 55, 258, 193, 326, 67, 95, 280, 330, 226, 247, 265, 320, 190, 339, 373, 271, 267, 250, 344, 282, 334, 221, 168, 315, 237, 383, 141, 316, 276, 353, 132, 113, 1, 277, 73, 357, 336, 324, 284, 368, 124, 147, 79, 65, 195, 295, 252, 197, 143, 76, 322, 309, 45, 248, 14, 152, 87, 298, 69, 135, 222, 266, 140, 125, 122, 299, 116, 74, 257, 355, 30, 26, 179, 380, 278, 161, 215, 101, 167, 184, 108, 40, 349, 269, 128, 75, 201, 261, 174, 28, 84, 352, 16, 92, 80, 204, 376, 335, 82, 7, 114, 20, 105, 319, 166, 148, 227, 164, 202, 230, 332, 187, 274, 38, 275, 41, 110, 160, 341, 71, 366, 303, 102, 162, 379, 129, 139, 259, 165, 169, 211, 233, 24, 48, 238, 155, 291, 2, 288, 264, 273, 170, 34, 374, 213, 85, 72, 328, 246, 111, 130, 346, 189, 311, 354, 300, 131, 178, 115, 27, 304, 121, 217, 154, 345, 363, 81, 350, 181, 290, 294, 15, 172, 171, 35, 359, 283, 314, 301, 31, 292, 272, 83, 337, 268, 9, 205, 293, 62, 331, 104, 70, 188, 323, 77, 88, 175, 239, 33, 208, 192, 11, 145, 351, 369, 340, 308, 317, 360, 44, 361, 186, 142, 371, 117, 333, 377, 21, 156, 133, 327, 318, 107, 253, 382, 57, 219, 58, 348, 260, 23, 191, 127, 347, 159, 157, 289, 134, 59, 279, 224, 381, 89, 98, 5, 66, 3, 229, 364, 206] + 28: [161, 180, 207, 215, 257, 279, 311, 329, 336, 338, 343, 353, 250, 190, 298, 359, 379, 10, 18, 60, 86, 115, 150, 156, 26, 67, 367, 14, 314, 15, 364, 303, 76, 145, 301, 173, 185, 59, 163, 376, 121, 205, 372, 348, 101, 181, 2, 132, 168, 346, 88, 240, 178, 6, 148, 31, 89, 174, 187, 9, 344, 7, 116, 32, 300, 327, 362, 154, 293, 126, 189, 138, 92, 307, 42, 167, 238, 278, 118, 66, 170, 72, 276, 147, 356, 253, 325, 135, 13, 380, 244, 210, 211, 85, 237, 11, 153, 113, 368, 285, 217, 16, 112, 322, 294, 38, 267, 309, 70, 214, 223, 339, 349, 247, 179, 61, 34, 225, 275, 313, 317, 93, 227, 28, 19, 48, 256, 270, 334, 146, 347, 40, 255, 122, 137, 90, 289, 290, 64, 159, 79, 96, 373, 318, 306, 288, 320, 361, 370, 282, 44, 69, 377, 80, 47, 186, 302, 103, 236, 383, 182, 316, 245, 106, 366, 284, 35, 304, 162, 233, 166, 200, 125, 134, 49, 143, 24, 351, 141, 371, 243, 172, 378, 352, 209, 268, 142, 258, 73, 39, 111, 53, 30, 195, 75, 228, 308, 193, 183, 12, 332, 8, 192, 212, 107, 155, 374, 261, 242, 222, 57, 82, 109, 119, 123, 254, 52, 97, 272, 158, 340, 43, 27, 104, 342, 231, 139, 262, 25, 120, 199, 5, 221, 341, 365, 169, 363, 234, 259, 62, 99, 176, 37, 58, 299, 68, 171, 305, 117, 149, 140, 354, 286, 226, 357, 165, 20, 114, 46, 175, 151, 110, 277, 360, 241, 152, 263, 204, 56, 127, 264, 36, 133, 131, 232, 198, 197, 203, 108, 51, 216, 315, 265, 296, 280, 246, 29, 319, 321, 188, 369, 71, 33, 273, 295, 375, 218, 213, 291, 83, 4, 328, 224, 100, 269, 333, 208, 381, 330, 266, 45, 202, 355, 283, 50, 129, 271, 206, 194, 230, 297, 292, 55, 87, 312, 324, 128, 157, 41, 184, 249, 21, 136, 251, 94, 350, 0, 22, 219, 63, 326, 1, 358, 177, 382, 345, 252, 124, 248, 77, 95, 235, 105, 335, 260, 74, 84, 17, 229, 54, 201, 196, 191, 160, 337, 78, 81, 98, 144, 287, 102, 65, 239, 310, 3, 130, 220, 91, 23, 274, 331, 323, 281, 164] + 29: [118, 324, 19, 205, 79, 199, 252, 8, 98, 355, 152, 176, 244, 4, 18, 105, 157, 84, 162, 234, 64, 212, 29, 66, 361, 123, 248, 380, 217, 111, 374, 251, 279, 367, 108, 91, 197, 175, 350, 378, 292, 167, 327, 288, 141, 189, 256, 26, 73, 87, 357, 85, 101, 359, 55, 261, 17, 286, 335, 240, 196, 16, 82, 329, 280, 312, 52, 356, 290, 201, 274, 322, 336, 208, 343, 172, 72, 313, 230, 153, 149, 254, 284, 363, 318, 202, 24, 132, 347, 269, 170, 7, 376, 49, 37, 39, 51, 320, 188, 61, 225, 125, 126, 158, 168, 325, 1, 285, 137, 309, 332, 364, 136, 20, 93, 70, 113, 334, 260, 81, 277, 186, 235, 94, 131, 32, 315, 42, 227, 30, 41, 143, 323, 74, 190, 278, 349, 146, 83, 377, 62, 0, 80, 138, 198, 262, 134, 50, 360, 142, 373, 381, 13, 306, 291, 107, 203, 110, 375, 145, 182, 104, 294, 103, 338, 43, 245, 241, 295, 33, 159, 68, 12, 181, 316, 209, 192, 246, 223, 144, 304, 265, 57, 216, 139, 59, 229, 147, 233, 296, 358, 249, 362, 339, 112, 207, 273, 264, 169, 302, 194, 71, 5, 268, 67, 129, 160, 102, 178, 228, 224, 28, 239, 297, 133, 281, 60, 333, 351, 293, 298, 353, 317, 287, 161, 219, 27, 342, 236, 258, 148, 38, 314, 267, 344, 114, 166, 99, 75, 165, 44, 354, 266, 319, 156, 171, 242, 369, 270, 154, 130, 221, 282, 120, 346, 382, 200, 300, 348, 40, 307, 263, 226, 321, 173, 379, 183, 193, 215, 345, 77, 213, 253, 92, 96, 6, 23, 76, 243, 163, 128, 78, 328, 368, 250, 308, 231, 372, 311, 25, 65, 86, 259, 3, 371, 56, 299, 109, 155, 177, 127, 54, 174, 370, 116, 151, 47, 53, 15, 45, 21, 289, 11, 365, 119, 341, 257, 276, 31, 10, 352, 164, 301, 185, 255, 124, 326, 210, 218, 310, 340, 220, 9, 115, 95, 305, 89, 195, 140, 237, 232, 90, 238, 303, 187, 184, 100, 117, 383, 58, 48, 63, 222, 97, 122, 191, 69, 337, 330, 247, 271, 88, 366, 275, 106, 272, 36, 46, 121, 35, 331, 214, 283, 14, 180, 204, 34, 150, 22, 135, 179, 211, 2, 206] + 30: [208, 189, 218, 219, 237, 259, 268, 269, 322, 334, 345, 380, 239, 40, 47, 88, 94, 101, 118, 123, 128, 134, 146, 185, 140, 305, 307, 331, 354, 363, 379, 8, 21, 22, 31, 36, 266, 374, 324, 273, 198, 56, 87, 70, 201, 227, 238, 301, 65, 336, 309, 104, 179, 349, 263, 304, 126, 300, 27, 131, 245, 74, 217, 356, 51, 297, 230, 69, 368, 229, 34, 30, 37, 362, 256, 157, 162, 375, 350, 90, 173, 80, 119, 176, 262, 271, 339, 260, 108, 280, 175, 194, 165, 178, 149, 191, 187, 28, 224, 195, 281, 310, 92, 337, 209, 16, 199, 251, 358, 234, 43, 299, 98, 114, 180, 220, 32, 190, 235, 252, 53, 188, 303, 71, 274, 38, 66, 145, 139, 117, 265, 4, 214, 361, 250, 152, 296, 167, 129, 290, 186, 327, 308, 313, 52, 11, 315, 100, 170, 84, 172, 284, 288, 370, 232, 270, 35, 206, 302, 6, 18, 261, 111, 79, 318, 130, 3, 63, 143, 2, 316, 1, 344, 353, 197, 44, 360, 82, 364, 171, 351, 295, 205, 177, 382, 278, 376, 50, 25, 5, 81, 367, 373, 168, 17, 253, 292, 91, 14, 248, 276, 85, 122, 148, 156, 257, 212, 357, 277, 96, 72, 109, 267, 161, 330, 338, 62, 46, 283, 12, 275, 200, 54, 286, 371, 57, 127, 26, 289, 24, 164, 221, 215, 365, 359, 241, 347, 326, 113, 112, 133, 225, 68, 77, 192, 174, 163, 211, 107, 264, 120, 207, 158, 378, 147, 317, 58, 23, 383, 55, 121, 348, 255, 102, 254, 59, 33, 89, 9, 132, 103, 293, 247, 203, 153, 61, 124, 78, 41, 341, 236, 325, 231, 332, 183, 279, 352, 106, 184, 196, 64, 97, 141, 298, 142, 48, 95, 83, 291, 93, 213, 135, 321, 49, 136, 369, 294, 7, 154, 125, 314, 182, 333, 242, 223, 312, 226, 285, 20, 249, 311, 155, 355, 228, 76, 75, 10, 216, 204, 160, 319, 282, 342, 287, 15, 328, 222, 366, 144, 159, 320, 67, 150, 73, 377, 99, 105, 372, 272, 243, 45, 240, 151, 60, 335, 244, 202, 381, 169, 39, 86, 13, 246, 138, 193, 343, 329, 115, 166, 323, 233, 42, 29, 137, 346, 340, 210, 258, 306, 181, 110, 116, 19, 0] + 31: [0, 246, 289, 294, 347, 172, 229, 120, 132, 207, 329, 350, 159, 49, 43, 375, 113, 313, 165, 89, 180, 37, 90, 149, 128, 81, 310, 20, 351, 99, 280, 297, 117, 203, 71, 212, 233, 189, 40, 15, 118, 12, 194, 367, 274, 18, 343, 148, 26, 34, 145, 243, 256, 195, 47, 42, 126, 167, 67, 341, 201, 206, 286, 380, 176, 97, 354, 107, 315, 52, 377, 200, 68, 186, 355, 254, 31, 374, 361, 103, 129, 127, 199, 154, 100, 317, 208, 231, 134, 383, 247, 352, 321, 7, 86, 13, 166, 339, 378, 33, 66, 41, 29, 323, 308, 64, 35, 30, 324, 267, 61, 223, 54, 287, 84, 272, 226, 327, 282, 242, 158, 270, 170, 296, 55, 202, 8, 266, 162, 234, 53, 163, 252, 95, 171, 106, 245, 316, 291, 177, 306, 300, 293, 94, 143, 187, 188, 290, 98, 190, 142, 59, 91, 340, 156, 218, 115, 215, 333, 258, 23, 77, 373, 51, 264, 160, 181, 257, 239, 147, 58, 269, 38, 318, 21, 196, 50, 298, 235, 311, 278, 25, 60, 112, 17, 222, 225, 73, 238, 85, 255, 27, 144, 250, 155, 348, 211, 65, 141, 376, 275, 262, 79, 332, 363, 69, 119, 214, 78, 122, 369, 6, 83, 337, 193, 359, 70, 335, 140, 334, 249, 46, 284, 260, 304, 157, 135, 151, 261, 326, 22, 102, 28, 88, 16, 271, 220, 9, 381, 104, 185, 133, 209, 62, 152, 277, 92, 273, 224, 210, 295, 174, 87, 364, 349, 276, 82, 182, 123, 183, 228, 178, 109, 5, 76, 116, 325, 253, 93, 175, 303, 237, 48, 309, 124, 114, 227, 292, 39, 131, 236, 205, 213, 4, 198, 139, 344, 179, 248, 121, 150, 72, 232, 366, 217, 362, 130, 345, 372, 301, 320, 379, 219, 360, 74, 368, 302, 108, 204, 24, 356, 10, 125, 279, 330, 56, 111, 322, 314, 153, 281, 101, 382, 161, 331, 36, 173, 283, 244, 105, 307, 110, 263, 259, 44, 319, 265, 3, 96, 338, 353, 137, 312, 285, 80, 240, 370, 184, 32, 365, 268, 164, 138, 169, 45, 14, 346, 342, 197, 357, 192, 241, 146, 75, 136, 57, 336, 11, 371, 168, 191, 230, 221, 19, 328, 299, 63, 2, 288, 305, 1, 251, 216, 358] + 32: [63, 177, 181, 244, 245, 251, 262, 265, 296, 312, 373, 375, 367, 316, 379, 57, 68, 147, 167, 173, 383, 14, 123, 169, 352, 382, 315, 320, 13, 118, 88, 11, 90, 145, 162, 300, 94, 283, 289, 69, 237, 276, 82, 261, 196, 93, 226, 114, 218, 184, 120, 111, 230, 4, 154, 124, 201, 170, 159, 333, 148, 174, 357, 71, 59, 80, 372, 103, 238, 277, 215, 235, 33, 202, 113, 104, 332, 115, 366, 243, 117, 224, 242, 267, 92, 219, 140, 301, 50, 127, 6, 5, 376, 23, 180, 15, 285, 274, 211, 189, 231, 256, 24, 339, 105, 364, 358, 328, 347, 257, 279, 7, 26, 293, 247, 110, 248, 1, 216, 27, 190, 12, 151, 306, 199, 32, 337, 34, 20, 278, 192, 54, 241, 2, 126, 345, 101, 142, 129, 144, 341, 31, 259, 326, 163, 121, 83, 294, 185, 253, 30, 172, 195, 0, 355, 194, 51, 360, 263, 58, 292, 182, 205, 311, 125, 305, 343, 272, 122, 66, 75, 252, 354, 86, 302, 363, 287, 365, 60, 260, 327, 280, 96, 19, 85, 314, 350, 349, 152, 197, 46, 321, 38, 8, 225, 228, 91, 213, 84, 309, 282, 44, 52, 318, 164, 200, 65, 156, 72, 28, 81, 10, 361, 119, 79, 207, 107, 370, 89, 87, 212, 250, 240, 378, 165, 304, 76, 368, 344, 21, 348, 131, 254, 48, 351, 236, 42, 270, 208, 112, 166, 130, 186, 331, 297, 178, 336, 317, 221, 330, 138, 291, 329, 271, 319, 97, 227, 269, 3, 381, 9, 374, 377, 98, 322, 310, 380, 116, 369, 45, 286, 158, 99, 264, 188, 217, 275, 229, 204, 78, 303, 25, 53, 193, 102, 134, 67, 295, 288, 179, 359, 143, 36, 175, 70, 22, 132, 73, 255, 47, 155, 323, 40, 258, 325, 64, 371, 340, 284, 153, 220, 17, 246, 106, 61, 308, 171, 157, 249, 62, 307, 356, 334, 74, 234, 187, 214, 233, 313, 150, 183, 139, 136, 35, 149, 290, 135, 209, 16, 335, 168, 281, 39, 56, 43, 128, 100, 95, 268, 362, 41, 273, 298, 324, 353, 198, 191, 222, 133, 18, 346, 146, 342, 109, 266, 239, 29, 77, 299, 223, 137, 161, 49, 108, 141, 232, 55, 206, 176, 37, 210, 160, 203, 338] + 33: [38, 32, 95, 102, 69, 142, 220, 277, 287, 293, 333, 372, 91, 311, 317, 114, 43, 135, 207, 250, 309, 264, 12, 34, 339, 151, 305, 79, 279, 15, 324, 332, 326, 185, 189, 161, 36, 335, 356, 187, 19, 122, 26, 238, 365, 228, 159, 129, 319, 37, 155, 351, 246, 105, 371, 194, 243, 275, 321, 83, 260, 314, 376, 308, 355, 373, 113, 128, 286, 306, 344, 236, 177, 73, 255, 131, 205, 156, 345, 29, 78, 341, 357, 265, 362, 342, 190, 192, 93, 256, 58, 166, 6, 346, 7, 313, 64, 82, 48, 117, 211, 375, 213, 3, 248, 188, 242, 46, 380, 183, 169, 237, 329, 262, 234, 71, 261, 119, 178, 143, 229, 39, 127, 208, 202, 162, 60, 281, 20, 74, 171, 44, 148, 107, 361, 227, 303, 196, 364, 215, 41, 225, 92, 57, 81, 297, 266, 193, 232, 123, 163, 336, 219, 27, 315, 47, 343, 96, 206, 210, 8, 276, 152, 249, 295, 184, 337, 203, 115, 379, 139, 25, 126, 347, 212, 49, 101, 312, 244, 223, 251, 325, 84, 33, 290, 216, 200, 350, 327, 291, 363, 302, 258, 231, 72, 1, 354, 383, 360, 24, 285, 62, 118, 239, 301, 121, 164, 30, 144, 254, 369, 137, 53, 310, 353, 263, 283, 323, 226, 150, 61, 80, 13, 170, 271, 120, 98, 272, 299, 158, 5, 338, 18, 45, 180, 330, 182, 111, 274, 99, 154, 42, 352, 87, 116, 198, 88, 106, 298, 181, 253, 269, 307, 31, 108, 160, 235, 124, 340, 368, 278, 245, 134, 0, 267, 153, 146, 218, 14, 22, 186, 292, 67, 132, 97, 172, 94, 377, 201, 35, 288, 4, 197, 70, 136, 85, 165, 296, 89, 191, 125, 273, 334, 149, 209, 349, 23, 300, 374, 359, 86, 280, 68, 233, 133, 173, 367, 28, 112, 257, 247, 282, 167, 331, 179, 270, 217, 174, 370, 66, 110, 224, 59, 77, 328, 259, 348, 21, 289, 230, 382, 52, 221, 168, 199, 304, 51, 63, 147, 141, 268, 90, 50, 56, 316, 130, 195, 2, 240, 320, 322, 54, 222, 138, 104, 366, 252, 381, 378, 284, 140, 145, 358, 204, 9, 16, 157, 103, 318, 76, 175, 100, 176, 75, 241, 17, 40, 109, 55, 65, 10, 11, 214, 294] + 34: [108, 76, 119, 122, 132, 160, 189, 193, 202, 229, 363, 365, 179, 377, 177, 198, 211, 213, 224, 279, 316, 341, 381, 14, 16, 10, 298, 282, 55, 171, 7, 309, 32, 66, 273, 343, 249, 87, 84, 336, 30, 322, 261, 289, 155, 112, 376, 375, 306, 92, 58, 37, 102, 301, 88, 100, 131, 351, 320, 121, 15, 2, 21, 327, 346, 263, 64, 226, 49, 355, 350, 129, 340, 97, 201, 123, 241, 50, 283, 275, 255, 104, 106, 235, 366, 79, 208, 175, 181, 5, 62, 268, 75, 9, 43, 303, 337, 295, 225, 313, 245, 35, 127, 56, 305, 297, 20, 369, 69, 284, 162, 256, 328, 367, 323, 139, 228, 278, 182, 378, 259, 220, 244, 12, 217, 48, 103, 45, 36, 370, 188, 38, 299, 347, 65, 345, 70, 270, 277, 149, 233, 173, 120, 94, 126, 178, 271, 184, 197, 247, 19, 221, 269, 250, 156, 234, 330, 362, 335, 107, 166, 204, 154, 51, 111, 260, 292, 276, 334, 240, 83, 91, 368, 236, 230, 85, 353, 176, 96, 252, 248, 293, 59, 95, 302, 33, 150, 114, 23, 314, 253, 90, 218, 243, 115, 77, 159, 67, 72, 308, 192, 307, 205, 130, 3, 321, 99, 4, 326, 195, 227, 157, 46, 374, 339, 212, 44, 185, 379, 125, 174, 251, 73, 180, 147, 161, 17, 257, 196, 25, 146, 6, 200, 280, 231, 294, 105, 13, 342, 1, 190, 223, 360, 267, 153, 133, 214, 318, 358, 262, 26, 22, 61, 145, 203, 11, 329, 60, 333, 232, 152, 331, 242, 163, 372, 304, 167, 288, 290, 187, 364, 361, 68, 34, 109, 216, 382, 354, 40, 8, 81, 183, 170, 42, 371, 310, 285, 71, 29, 158, 138, 238, 210, 24, 89, 215, 98, 93, 117, 325, 207, 239, 148, 191, 356, 199, 359, 352, 264, 265, 165, 344, 47, 82, 291, 254, 53, 136, 116, 80, 266, 86, 319, 338, 237, 311, 31, 41, 110, 143, 209, 113, 272, 274, 52, 54, 151, 135, 315, 168, 134, 172, 0, 140, 383, 286, 57, 128, 324, 74, 246, 300, 124, 317, 27, 164, 186, 101, 380, 349, 357, 312, 258, 78, 144, 137, 169, 348, 296, 118, 28, 281, 332, 18, 194, 222, 373, 206, 287, 63, 142, 219, 39, 141] + 35: [319, 90, 91, 196, 105, 207, 225, 336, 359, 228, 177, 343, 45, 163, 192, 11, 82, 344, 195, 183, 124, 146, 60, 293, 1, 46, 98, 5, 32, 369, 66, 135, 354, 193, 0, 13, 345, 364, 27, 9, 372, 172, 19, 285, 262, 10, 212, 381, 363, 15, 159, 267, 209, 111, 383, 292, 253, 310, 51, 40, 75, 355, 52, 357, 95, 350, 185, 332, 29, 107, 120, 291, 169, 4, 362, 272, 108, 96, 127, 73, 250, 238, 229, 304, 28, 138, 265, 373, 137, 57, 7, 25, 94, 101, 173, 284, 205, 47, 311, 275, 93, 302, 117, 84, 181, 148, 251, 200, 299, 257, 114, 157, 282, 256, 264, 175, 189, 288, 219, 59, 218, 141, 375, 312, 171, 12, 106, 17, 248, 77, 123, 242, 131, 55, 352, 37, 61, 298, 348, 382, 365, 316, 178, 341, 297, 286, 41, 329, 308, 215, 323, 301, 217, 23, 165, 283, 367, 314, 186, 54, 134, 300, 227, 24, 338, 249, 197, 71, 295, 318, 224, 222, 187, 243, 145, 216, 63, 259, 371, 188, 68, 67, 78, 331, 296, 313, 202, 34, 374, 306, 271, 76, 133, 368, 239, 126, 102, 122, 287, 142, 279, 49, 255, 100, 53, 56, 147, 136, 321, 152, 74, 99, 378, 26, 263, 85, 86, 211, 289, 50, 244, 42, 109, 376, 6, 115, 69, 39, 342, 254, 3, 241, 325, 334, 235, 179, 35, 232, 351, 119, 203, 379, 221, 326, 132, 268, 110, 339, 103, 112, 358, 118, 276, 303, 231, 335, 18, 294, 170, 269, 79, 377, 233, 208, 72, 58, 346, 234, 81, 328, 214, 347, 176, 149, 83, 210, 322, 130, 154, 361, 360, 337, 8, 273, 87, 161, 33, 252, 113, 237, 206, 191, 128, 30, 16, 70, 315, 89, 97, 199, 281, 144, 116, 194, 36, 104, 305, 330, 198, 245, 270, 247, 340, 62, 31, 164, 230, 190, 180, 204, 274, 278, 327, 226, 48, 240, 38, 307, 158, 213, 261, 380, 43, 121, 167, 64, 309, 182, 129, 162, 168, 201, 92, 184, 317, 22, 139, 236, 80, 246, 153, 266, 160, 125, 20, 290, 151, 156, 333, 280, 353, 44, 324, 349, 140, 143, 277, 174, 88, 155, 150, 2, 220, 260, 21, 14, 356, 320, 366, 166, 65, 370, 223, 258] + 36: [232, 64, 127, 313, 336, 37, 123, 170, 273, 302, 360, 375, 22, 171, 310, 157, 135, 209, 253, 26, 317, 145, 292, 25, 184, 138, 289, 343, 36, 149, 383, 120, 124, 306, 279, 16, 371, 233, 293, 284, 195, 71, 220, 265, 137, 227, 367, 290, 368, 5, 108, 91, 177, 200, 243, 158, 14, 128, 214, 186, 272, 133, 309, 132, 354, 211, 244, 44, 337, 303, 377, 270, 53, 152, 321, 167, 153, 323, 165, 239, 19, 126, 114, 376, 340, 73, 162, 288, 20, 117, 76, 318, 259, 110, 203, 160, 341, 352, 314, 49, 105, 228, 212, 159, 189, 226, 96, 218, 144, 369, 85, 319, 322, 216, 206, 148, 173, 35, 141, 221, 57, 326, 31, 219, 342, 100, 264, 48, 70, 61, 1, 181, 304, 320, 93, 166, 50, 224, 0, 263, 198, 60, 269, 234, 238, 242, 47, 345, 95, 116, 255, 356, 190, 254, 285, 222, 266, 296, 92, 139, 297, 102, 305, 58, 329, 8, 235, 43, 10, 196, 205, 334, 122, 364, 180, 18, 363, 87, 378, 113, 325, 98, 79, 262, 103, 237, 39, 163, 6, 215, 286, 182, 311, 258, 308, 201, 63, 346, 134, 88, 281, 249, 161, 268, 178, 210, 150, 240, 13, 300, 280, 335, 188, 77, 256, 4, 193, 230, 301, 33, 67, 155, 140, 257, 208, 204, 324, 34, 225, 260, 56, 65, 9, 183, 366, 38, 267, 248, 316, 106, 372, 299, 40, 331, 246, 15, 147, 81, 23, 338, 115, 86, 12, 101, 274, 42, 194, 164, 202, 187, 29, 24, 312, 151, 223, 328, 241, 45, 362, 72, 174, 143, 80, 236, 21, 353, 107, 2, 373, 156, 112, 276, 11, 52, 283, 247, 350, 278, 131, 59, 380, 32, 94, 245, 46, 229, 365, 295, 355, 154, 192, 62, 90, 175, 111, 339, 179, 344, 119, 315, 307, 327, 252, 359, 99, 261, 231, 298, 41, 104, 51, 83, 213, 7, 294, 185, 374, 54, 282, 121, 277, 217, 146, 109, 347, 136, 197, 348, 97, 287, 361, 3, 142, 17, 351, 251, 130, 82, 370, 250, 207, 125, 55, 172, 271, 28, 358, 199, 357, 89, 382, 84, 275, 332, 349, 69, 168, 379, 129, 381, 75, 330, 74, 176, 291, 333, 191, 68, 66, 30, 78, 169, 118, 27] + 37: [218, 120, 226, 8, 75, 112, 132, 236, 239, 288, 352, 383, 40, 377, 67, 84, 99, 209, 251, 263, 31, 189, 56, 375, 268, 287, 376, 334, 105, 256, 62, 214, 275, 232, 238, 276, 294, 322, 6, 304, 380, 317, 60, 295, 307, 367, 23, 25, 148, 328, 19, 203, 161, 244, 271, 137, 183, 369, 12, 358, 293, 129, 118, 144, 111, 83, 4, 368, 96, 70, 211, 50, 109, 171, 93, 320, 296, 356, 373, 298, 291, 130, 186, 68, 301, 140, 235, 127, 43, 92, 128, 102, 260, 63, 107, 282, 379, 86, 318, 359, 21, 103, 55, 42, 91, 142, 237, 346, 58, 223, 353, 277, 173, 264, 207, 279, 286, 297, 159, 90, 158, 292, 182, 257, 168, 141, 347, 212, 332, 310, 250, 381, 222, 179, 302, 343, 229, 338, 126, 313, 26, 378, 324, 246, 200, 272, 47, 194, 316, 61, 87, 195, 172, 175, 231, 2, 366, 48, 355, 181, 85, 82, 100, 312, 249, 169, 57, 89, 79, 190, 344, 233, 208, 230, 278, 247, 354, 10, 35, 20, 349, 372, 114, 117, 336, 365, 285, 136, 5, 13, 106, 202, 162, 341, 156, 204, 370, 217, 333, 331, 210, 299, 146, 227, 113, 362, 16, 147, 145, 51, 135, 206, 267, 110, 166, 281, 340, 289, 59, 65, 345, 258, 329, 69, 325, 17, 101, 180, 0, 363, 361, 139, 164, 104, 134, 49, 187, 124, 248, 326, 188, 46, 174, 327, 153, 149, 201, 335, 154, 138, 225, 300, 240, 199, 382, 88, 330, 64, 39, 37, 33, 53, 259, 11, 319, 205, 77, 185, 315, 228, 34, 262, 284, 350, 283, 45, 71, 178, 371, 22, 32, 1, 41, 269, 24, 152, 339, 54, 364, 52, 234, 165, 131, 74, 143, 121, 274, 215, 374, 220, 66, 306, 95, 14, 163, 323, 30, 98, 94, 170, 81, 28, 273, 125, 193, 253, 150, 76, 3, 337, 280, 266, 191, 119, 123, 151, 72, 7, 133, 27, 108, 224, 351, 254, 357, 155, 243, 314, 290, 252, 18, 196, 73, 255, 97, 216, 198, 177, 265, 213, 44, 9, 192, 176, 36, 321, 38, 303, 308, 197, 348, 311, 305, 245, 221, 270, 29, 167, 116, 160, 184, 261, 242, 309, 342, 360, 157, 122, 15, 80, 241, 78, 115, 219] + 38: [301, 52, 101, 191, 210, 46, 60, 185, 237, 251, 325, 380, 6, 121, 304, 312, 64, 98, 159, 161, 176, 339, 340, 38, 266, 125, 193, 142, 170, 250, 11, 231, 18, 232, 270, 346, 203, 169, 341, 224, 367, 84, 310, 375, 202, 281, 330, 318, 158, 373, 234, 95, 359, 260, 376, 104, 25, 163, 175, 243, 378, 32, 327, 355, 96, 228, 149, 200, 183, 24, 209, 133, 221, 220, 206, 90, 362, 368, 81, 296, 247, 364, 128, 370, 269, 309, 135, 361, 146, 349, 188, 353, 43, 295, 316, 127, 280, 194, 328, 106, 352, 136, 265, 181, 302, 102, 164, 160, 124, 248, 317, 225, 99, 308, 190, 356, 100, 311, 236, 155, 201, 274, 212, 16, 219, 156, 117, 26, 261, 204, 314, 291, 284, 115, 279, 97, 73, 255, 34, 139, 74, 61, 299, 1, 372, 226, 0, 338, 14, 307, 141, 134, 129, 80, 343, 140, 51, 322, 278, 53, 9, 150, 241, 28, 347, 215, 42, 20, 30, 285, 166, 268, 72, 36, 276, 272, 238, 13, 240, 198, 366, 354, 17, 54, 321, 211, 111, 235, 56, 252, 259, 132, 189, 263, 130, 137, 275, 165, 246, 258, 334, 35, 91, 303, 126, 86, 19, 108, 208, 45, 5, 180, 273, 44, 113, 22, 383, 8, 217, 119, 271, 58, 48, 40, 360, 55, 254, 218, 114, 292, 92, 344, 21, 262, 148, 41, 182, 351, 320, 315, 154, 332, 27, 267, 85, 323, 298, 78, 79, 205, 326, 365, 277, 123, 381, 253, 227, 57, 179, 152, 214, 186, 357, 162, 333, 94, 145, 300, 239, 144, 342, 174, 337, 290, 335, 377, 89, 118, 244, 77, 29, 87, 229, 2, 23, 71, 65, 153, 62, 66, 49, 319, 69, 88, 7, 15, 313, 283, 12, 83, 282, 245, 293, 286, 222, 264, 151, 242, 122, 168, 70, 110, 10, 345, 63, 288, 147, 184, 143, 103, 195, 374, 192, 196, 4, 348, 171, 233, 107, 256, 109, 305, 172, 294, 363, 249, 187, 178, 230, 329, 75, 216, 31, 76, 120, 82, 350, 207, 223, 369, 39, 331, 67, 105, 289, 33, 173, 138, 112, 116, 93, 50, 379, 199, 324, 68, 37, 131, 371, 297, 3, 336, 213, 287, 177, 257, 59, 358, 306, 197, 382, 47, 157, 167] + 39: [282, 39, 262, 261, 321, 304, 143, 149, 257, 22, 44, 209, 310, 302, 230, 120, 107, 339, 57, 201, 52, 9, 198, 361, 247, 138, 317, 106, 306, 245, 64, 342, 269, 380, 165, 89, 66, 248, 378, 281, 320, 4, 288, 175, 147, 356, 29, 334, 104, 81, 340, 358, 33, 275, 70, 20, 370, 308, 90, 21, 372, 208, 127, 101, 366, 45, 6, 335, 211, 351, 154, 11, 236, 163, 352, 190, 371, 95, 16, 287, 83, 67, 73, 115, 56, 374, 87, 54, 144, 167, 114, 216, 197, 219, 332, 148, 315, 333, 23, 48, 353, 134, 128, 60, 117, 359, 184, 186, 258, 174, 121, 146, 47, 177, 285, 183, 283, 10, 238, 344, 286, 119, 26, 7, 232, 126, 14, 357, 265, 159, 229, 316, 65, 267, 71, 51, 140, 367, 345, 354, 207, 291, 92, 284, 312, 242, 324, 365, 191, 152, 301, 97, 30, 346, 206, 125, 38, 279, 277, 369, 274, 69, 135, 88, 164, 19, 157, 237, 93, 325, 204, 76, 170, 74, 188, 185, 221, 41, 108, 124, 132, 360, 296, 178, 259, 34, 276, 193, 254, 111, 337, 318, 36, 13, 362, 49, 383, 137, 199, 133, 79, 103, 266, 12, 314, 218, 270, 3, 42, 239, 322, 289, 195, 303, 305, 294, 313, 139, 297, 77, 8, 225, 63, 61, 309, 168, 55, 105, 379, 215, 244, 40, 375, 31, 75, 373, 327, 343, 253, 233, 85, 113, 86, 141, 235, 278, 179, 182, 130, 196, 246, 256, 43, 151, 80, 172, 298, 336, 5, 293, 169, 91, 252, 136, 100, 156, 292, 250, 290, 260, 62, 203, 220, 192, 160, 158, 176, 153, 382, 330, 227, 323, 162, 72, 363, 27, 347, 205, 99, 25, 272, 319, 118, 122, 213, 212, 68, 58, 142, 349, 96, 1, 155, 24, 53, 299, 249, 102, 194, 171, 0, 210, 280, 368, 271, 341, 84, 112, 364, 311, 181, 166, 123, 326, 35, 348, 328, 300, 273, 28, 161, 228, 240, 109, 18, 129, 180, 223, 187, 15, 202, 251, 226, 355, 214, 376, 37, 255, 268, 82, 2, 98, 200, 331, 150, 241, 338, 189, 78, 224, 264, 234, 329, 17, 59, 131, 50, 173, 243, 222, 46, 32, 110, 116, 145, 295, 381, 263, 307, 94, 377, 217, 231, 350] + 40: [31, 249, 343, 376, 92, 105, 123, 149, 194, 214, 248, 300, 319, 308, 365, 29, 350, 0, 40, 252, 72, 312, 69, 170, 15, 334, 53, 155, 19, 116, 140, 325, 272, 240, 144, 353, 68, 296, 193, 187, 2, 171, 6, 186, 173, 154, 191, 331, 175, 258, 10, 382, 43, 329, 139, 23, 113, 255, 225, 344, 378, 198, 136, 38, 41, 120, 131, 288, 134, 39, 320, 5, 317, 208, 89, 145, 309, 165, 304, 118, 361, 34, 234, 220, 207, 251, 143, 30, 79, 7, 195, 147, 133, 104, 315, 20, 183, 81, 363, 211, 44, 306, 93, 238, 141, 285, 380, 127, 250, 64, 1, 253, 205, 290, 327, 36, 78, 111, 267, 83, 349, 164, 275, 289, 236, 150, 243, 301, 32, 374, 172, 335, 360, 232, 311, 162, 196, 110, 264, 261, 135, 66, 157, 126, 142, 26, 3, 271, 269, 65, 305, 314, 49, 16, 298, 146, 328, 160, 224, 168, 216, 263, 338, 293, 125, 203, 223, 197, 71, 297, 58, 129, 278, 316, 294, 273, 270, 50, 180, 24, 358, 247, 235, 336, 200, 99, 192, 37, 122, 368, 114, 42, 190, 52, 74, 284, 54, 56, 355, 48, 51, 268, 230, 245, 21, 337, 372, 362, 166, 307, 94, 318, 291, 115, 231, 313, 108, 265, 332, 9, 11, 137, 103, 167, 201, 84, 295, 45, 179, 283, 369, 303, 27, 259, 151, 371, 76, 212, 22, 352, 33, 62, 254, 159, 322, 370, 57, 323, 222, 184, 163, 381, 346, 101, 158, 210, 341, 302, 130, 287, 241, 14, 217, 87, 226, 98, 88, 28, 277, 221, 189, 333, 233, 266, 77, 109, 132, 75, 153, 347, 61, 199, 244, 356, 345, 102, 117, 86, 379, 367, 8, 91, 185, 12, 364, 237, 18, 310, 257, 176, 357, 373, 156, 59, 219, 46, 178, 281, 366, 124, 174, 112, 340, 67, 215, 17, 63, 148, 95, 330, 73, 276, 35, 96, 280, 55, 279, 228, 321, 229, 299, 206, 47, 383, 218, 152, 260, 204, 161, 97, 274, 262, 82, 282, 85, 348, 242, 342, 286, 359, 107, 90, 326, 239, 138, 177, 60, 13, 182, 292, 169, 213, 202, 100, 80, 188, 25, 339, 375, 121, 119, 354, 181, 70, 128, 351, 209, 227, 377, 324, 246, 256, 106, 4] + 41: [49, 64, 325, 181, 221, 148, 373, 341, 227, 216, 309, 357, 378, 222, 276, 339, 10, 214, 98, 137, 193, 135, 22, 278, 199, 326, 134, 271, 299, 208, 352, 141, 17, 343, 152, 85, 170, 280, 324, 254, 242, 315, 303, 147, 28, 16, 164, 178, 60, 360, 53, 116, 289, 338, 342, 13, 20, 2, 247, 132, 350, 161, 351, 345, 167, 186, 99, 105, 204, 58, 230, 296, 374, 314, 4, 301, 115, 290, 188, 72, 209, 304, 333, 31, 321, 358, 318, 26, 229, 63, 81, 1, 287, 110, 372, 263, 269, 34, 253, 264, 258, 144, 371, 123, 248, 265, 382, 228, 298, 187, 104, 163, 56, 66, 153, 75, 207, 375, 169, 118, 283, 7, 383, 353, 282, 380, 43, 15, 68, 47, 157, 120, 114, 365, 95, 202, 294, 250, 173, 37, 48, 14, 109, 52, 142, 200, 279, 35, 102, 117, 295, 272, 259, 29, 346, 233, 355, 168, 24, 3, 32, 340, 107, 128, 191, 175, 323, 40, 184, 84, 69, 127, 349, 322, 101, 18, 154, 234, 67, 361, 190, 273, 21, 45, 174, 235, 176, 19, 292, 297, 238, 362, 30, 86, 364, 108, 126, 54, 151, 284, 300, 38, 195, 291, 369, 57, 111, 44, 219, 55, 112, 244, 41, 149, 171, 145, 197, 150, 172, 9, 140, 256, 224, 61, 89, 310, 231, 240, 366, 255, 79, 80, 36, 46, 268, 277, 179, 320, 245, 319, 129, 249, 97, 334, 218, 125, 51, 246, 39, 381, 121, 356, 275, 90, 76, 302, 308, 379, 8, 130, 78, 0, 62, 88, 159, 212, 92, 347, 217, 225, 330, 213, 59, 158, 274, 177, 119, 307, 96, 206, 252, 313, 131, 6, 33, 65, 139, 189, 215, 27, 368, 306, 286, 156, 138, 74, 146, 73, 155, 203, 205, 376, 223, 82, 42, 237, 344, 220, 354, 113, 182, 281, 305, 251, 293, 262, 336, 70, 363, 91, 143, 77, 183, 211, 122, 11, 285, 311, 194, 93, 133, 160, 100, 165, 201, 270, 185, 71, 136, 198, 243, 241, 210, 261, 236, 180, 267, 162, 87, 166, 50, 348, 25, 83, 266, 94, 232, 257, 329, 312, 367, 103, 327, 337, 239, 260, 288, 5, 192, 316, 370, 377, 335, 196, 124, 12, 23, 226, 328, 317, 359, 332, 331, 106] + 42: [123, 193, 242, 294, 369, 376, 25, 71, 95, 254, 324, 339, 152, 313, 213, 285, 115, 13, 365, 24, 252, 289, 46, 27, 117, 232, 233, 175, 134, 331, 86, 361, 273, 352, 126, 121, 31, 259, 356, 160, 180, 7, 343, 292, 353, 15, 163, 40, 359, 209, 4, 103, 166, 371, 58, 346, 82, 136, 159, 66, 217, 255, 320, 161, 257, 311, 298, 364, 90, 304, 182, 74, 94, 208, 277, 287, 314, 192, 97, 378, 360, 186, 14, 91, 266, 263, 358, 230, 322, 137, 64, 53, 98, 330, 78, 240, 268, 41, 189, 178, 147, 362, 372, 205, 26, 229, 22, 190, 345, 179, 281, 341, 338, 227, 231, 310, 120, 261, 96, 276, 176, 150, 327, 221, 329, 39, 237, 258, 112, 199, 383, 44, 172, 264, 342, 29, 326, 76, 307, 34, 293, 267, 302, 12, 155, 282, 203, 30, 216, 149, 62, 299, 194, 272, 133, 35, 218, 72, 59, 354, 167, 38, 297, 226, 156, 197, 144, 54, 93, 63, 48, 248, 236, 316, 132, 246, 270, 321, 174, 47, 325, 215, 55, 379, 201, 335, 340, 139, 336, 104, 124, 382, 111, 151, 220, 235, 173, 36, 241, 283, 68, 102, 207, 249, 256, 279, 374, 265, 344, 234, 18, 260, 16, 363, 3, 177, 158, 350, 168, 21, 87, 92, 60, 315, 125, 290, 153, 11, 333, 116, 243, 373, 286, 223, 381, 183, 171, 56, 69, 70, 138, 212, 162, 122, 239, 366, 146, 323, 170, 284, 85, 57, 37, 247, 222, 127, 67, 280, 301, 9, 204, 73, 296, 0, 2, 211, 337, 319, 165, 181, 51, 89, 61, 185, 309, 100, 188, 305, 200, 348, 105, 308, 8, 219, 269, 318, 88, 110, 210, 184, 349, 191, 28, 143, 380, 157, 251, 145, 198, 271, 154, 303, 262, 275, 17, 77, 32, 244, 291, 250, 288, 43, 375, 106, 108, 131, 135, 33, 80, 49, 332, 368, 334, 142, 20, 357, 306, 370, 169, 45, 101, 238, 196, 312, 367, 79, 1, 119, 377, 274, 140, 118, 113, 130, 355, 10, 23, 317, 50, 228, 6, 141, 224, 52, 245, 109, 5, 128, 75, 81, 42, 114, 129, 328, 214, 164, 65, 99, 278, 206, 347, 83, 295, 148, 195, 351, 187, 202, 19, 225, 300, 84, 253, 107] + 43: [168, 374, 234, 293, 174, 116, 211, 317, 302, 195, 354, 3, 225, 264, 88, 306, 229, 325, 215, 258, 69, 104, 214, 245, 375, 322, 255, 200, 165, 226, 175, 260, 248, 176, 44, 348, 66, 186, 280, 327, 130, 107, 253, 163, 151, 7, 212, 95, 98, 377, 136, 32, 34, 345, 120, 65, 109, 41, 25, 182, 315, 363, 224, 73, 261, 273, 266, 193, 158, 106, 244, 331, 221, 97, 127, 238, 26, 242, 128, 294, 265, 124, 289, 373, 105, 254, 342, 368, 86, 24, 101, 207, 110, 285, 54, 169, 171, 20, 301, 250, 219, 185, 146, 329, 187, 311, 189, 192, 156, 13, 138, 85, 83, 241, 45, 292, 240, 162, 361, 206, 347, 349, 111, 379, 339, 196, 217, 170, 357, 276, 319, 382, 92, 39, 223, 321, 286, 28, 180, 288, 11, 290, 114, 139, 372, 216, 29, 89, 204, 67, 210, 18, 72, 300, 312, 376, 335, 37, 55, 121, 48, 157, 183, 370, 64, 119, 172, 350, 17, 364, 153, 78, 275, 160, 49, 16, 117, 118, 328, 91, 359, 59, 81, 147, 205, 115, 113, 256, 295, 179, 51, 310, 324, 112, 93, 198, 378, 123, 50, 80, 159, 316, 201, 33, 14, 35, 337, 178, 380, 277, 47, 343, 31, 209, 0, 333, 283, 197, 291, 108, 222, 140, 38, 309, 122, 320, 62, 58, 190, 129, 235, 307, 272, 36, 243, 284, 353, 134, 137, 133, 75, 371, 152, 305, 150, 61, 10, 46, 8, 173, 249, 15, 30, 142, 57, 360, 282, 257, 155, 56, 76, 287, 21, 383, 297, 218, 131, 227, 154, 84, 194, 167, 71, 233, 141, 42, 298, 68, 103, 1, 2, 149, 145, 213, 279, 330, 228, 208, 43, 278, 94, 338, 4, 365, 334, 74, 356, 355, 341, 166, 351, 308, 366, 269, 77, 79, 237, 70, 87, 381, 236, 270, 23, 332, 314, 318, 52, 367, 148, 125, 259, 188, 199, 184, 164, 177, 239, 323, 346, 296, 12, 143, 126, 247, 281, 340, 369, 161, 252, 202, 232, 135, 60, 9, 304, 336, 53, 96, 27, 6, 100, 274, 191, 5, 63, 251, 132, 19, 230, 231, 344, 181, 22, 203, 246, 271, 99, 263, 144, 268, 303, 352, 262, 82, 326, 313, 90, 299, 102, 220, 267, 358, 362, 40] + 44: [213, 259, 133, 209, 129, 332, 303, 12, 106, 116, 322, 374, 77, 121, 100, 87, 370, 78, 234, 27, 246, 31, 281, 206, 301, 358, 138, 315, 44, 282, 218, 62, 373, 127, 75, 339, 191, 306, 265, 287, 221, 185, 113, 321, 115, 73, 310, 13, 199, 63, 207, 364, 177, 334, 179, 256, 195, 361, 181, 189, 10, 264, 276, 192, 269, 107, 309, 357, 245, 176, 52, 308, 317, 187, 144, 349, 130, 196, 105, 18, 273, 260, 111, 161, 226, 98, 228, 173, 164, 15, 149, 345, 312, 268, 316, 295, 348, 313, 304, 219, 366, 126, 243, 53, 124, 247, 353, 112, 362, 146, 217, 254, 160, 162, 336, 314, 35, 194, 58, 371, 274, 93, 125, 296, 380, 46, 19, 263, 153, 283, 76, 16, 172, 34, 48, 266, 71, 134, 40, 169, 232, 50, 293, 197, 171, 288, 33, 244, 70, 381, 159, 211, 7, 251, 37, 208, 222, 29, 320, 275, 186, 338, 344, 59, 241, 375, 330, 88, 379, 82, 143, 102, 184, 352, 92, 205, 367, 255, 210, 148, 101, 376, 166, 365, 6, 285, 36, 74, 20, 137, 119, 297, 223, 257, 30, 198, 262, 39, 261, 90, 305, 183, 151, 356, 284, 84, 236, 329, 91, 56, 369, 165, 3, 49, 5, 152, 21, 267, 11, 142, 117, 201, 120, 83, 104, 51, 22, 335, 145, 215, 325, 38, 340, 150, 96, 41, 231, 57, 80, 302, 170, 188, 229, 249, 95, 290, 248, 354, 347, 72, 200, 307, 346, 227, 311, 118, 360, 163, 131, 108, 214, 4, 65, 99, 168, 180, 331, 182, 89, 85, 54, 327, 81, 203, 216, 47, 238, 190, 135, 383, 355, 24, 286, 26, 270, 204, 114, 240, 97, 60, 279, 212, 343, 55, 66, 136, 318, 230, 1, 298, 64, 271, 252, 294, 291, 122, 132, 278, 8, 250, 326, 363, 167, 155, 43, 289, 94, 350, 174, 128, 156, 193, 253, 123, 0, 351, 368, 86, 337, 154, 79, 147, 323, 319, 45, 372, 17, 110, 277, 333, 324, 61, 378, 220, 377, 9, 2, 28, 258, 382, 328, 242, 225, 299, 67, 157, 341, 109, 42, 69, 25, 300, 140, 280, 14, 224, 359, 292, 23, 237, 239, 139, 272, 178, 68, 141, 158, 175, 202, 342, 32, 235, 233, 103] + 45: [92, 189, 195, 117, 192, 64, 266, 300, 100, 174, 221, 347, 140, 16, 357, 365, 34, 273, 220, 337, 295, 240, 336, 138, 178, 380, 51, 41, 123, 87, 263, 268, 125, 13, 181, 83, 10, 272, 158, 80, 2, 116, 333, 15, 321, 305, 69, 355, 227, 124, 310, 112, 175, 344, 294, 381, 243, 56, 66, 185, 154, 212, 307, 313, 279, 330, 9, 151, 238, 62, 130, 72, 23, 165, 168, 328, 106, 35, 264, 374, 362, 270, 259, 299, 121, 298, 118, 250, 260, 43, 358, 79, 371, 193, 366, 361, 161, 160, 183, 155, 213, 4, 205, 255, 312, 132, 53, 244, 293, 369, 107, 229, 360, 302, 46, 231, 309, 102, 148, 137, 42, 375, 52, 36, 345, 1, 230, 241, 203, 90, 101, 17, 334, 247, 209, 115, 135, 96, 170, 379, 48, 323, 352, 14, 201, 339, 224, 326, 267, 24, 126, 211, 289, 332, 265, 40, 8, 356, 128, 45, 210, 236, 207, 322, 119, 111, 194, 235, 27, 141, 349, 122, 7, 208, 133, 282, 341, 214, 200, 49, 33, 176, 297, 196, 251, 67, 76, 37, 342, 377, 186, 75, 350, 171, 114, 162, 306, 109, 20, 5, 129, 71, 31, 184, 367, 120, 348, 55, 327, 315, 19, 283, 177, 3, 82, 99, 373, 274, 163, 223, 68, 26, 65, 98, 57, 136, 153, 149, 276, 197, 61, 239, 84, 32, 58, 296, 142, 78, 97, 311, 329, 182, 21, 187, 54, 292, 340, 29, 169, 225, 280, 60, 164, 346, 269, 198, 308, 94, 50, 150, 127, 93, 382, 325, 95, 199, 131, 359, 12, 179, 218, 320, 271, 202, 152, 172, 81, 139, 180, 254, 70, 217, 370, 38, 288, 113, 143, 6, 383, 25, 47, 228, 372, 44, 303, 291, 166, 147, 167, 77, 324, 284, 103, 331, 245, 156, 261, 59, 285, 73, 18, 237, 191, 317, 354, 275, 74, 108, 376, 319, 246, 368, 226, 287, 173, 110, 286, 351, 277, 242, 222, 145, 204, 190, 232, 335, 301, 316, 85, 39, 104, 86, 22, 290, 363, 91, 89, 11, 318, 144, 378, 258, 88, 281, 253, 256, 105, 364, 353, 28, 219, 134, 262, 234, 30, 249, 257, 252, 314, 215, 159, 216, 233, 188, 146, 248, 338, 157, 63, 304, 0, 343, 206, 278] + 46: [142, 95, 321, 290, 159, 15, 358, 14, 270, 252, 88, 356, 183, 155, 50, 292, 318, 7, 337, 330, 28, 323, 325, 302, 352, 115, 156, 268, 92, 364, 305, 87, 134, 61, 122, 175, 311, 55, 211, 353, 294, 32, 255, 265, 26, 366, 167, 365, 133, 254, 85, 232, 304, 336, 78, 241, 93, 196, 178, 82, 10, 60, 39, 326, 109, 57, 206, 291, 339, 288, 174, 213, 247, 124, 242, 83, 4, 377, 263, 264, 375, 229, 179, 27, 34, 209, 219, 143, 164, 205, 223, 277, 347, 11, 248, 36, 64, 6, 303, 8, 42, 59, 144, 333, 309, 269, 225, 147, 139, 324, 18, 62, 118, 272, 161, 243, 52, 21, 185, 379, 256, 332, 345, 108, 197, 273, 173, 56, 135, 94, 276, 5, 58, 221, 113, 131, 315, 172, 313, 322, 150, 75, 47, 220, 286, 274, 338, 280, 168, 128, 307, 382, 331, 381, 54, 231, 380, 246, 114, 228, 121, 53, 81, 346, 283, 98, 281, 123, 253, 239, 165, 49, 132, 162, 193, 289, 359, 361, 1, 182, 355, 191, 378, 351, 245, 23, 158, 328, 91, 285, 107, 349, 343, 238, 204, 30, 218, 41, 45, 35, 261, 171, 341, 0, 110, 202, 38, 282, 334, 362, 111, 373, 190, 234, 31, 181, 207, 51, 68, 215, 99, 298, 103, 299, 20, 86, 73, 251, 149, 96, 40, 3, 130, 63, 198, 89, 317, 138, 77, 84, 230, 16, 187, 102, 195, 176, 237, 297, 69, 67, 127, 100, 367, 25, 136, 350, 33, 129, 216, 119, 293, 208, 258, 151, 354, 170, 271, 266, 22, 316, 340, 169, 74, 257, 72, 306, 106, 249, 194, 374, 260, 192, 383, 141, 90, 184, 105, 212, 319, 46, 104, 320, 80, 259, 17, 24, 348, 65, 43, 70, 66, 240, 344, 76, 186, 295, 101, 308, 177, 262, 363, 279, 140, 327, 12, 2, 372, 369, 125, 97, 145, 370, 48, 148, 335, 224, 154, 44, 287, 180, 157, 360, 342, 244, 126, 296, 152, 368, 284, 79, 117, 13, 250, 189, 71, 112, 275, 200, 188, 314, 312, 199, 227, 137, 226, 329, 217, 278, 236, 29, 210, 19, 301, 203, 376, 37, 235, 160, 9, 163, 310, 357, 201, 233, 214, 146, 120, 166, 267, 222, 153, 371, 300, 116] + 47: [195, 146, 191, 373, 209, 369, 246, 293, 360, 56, 288, 326, 3, 252, 303, 32, 275, 247, 165, 261, 113, 7, 236, 62, 343, 50, 112, 335, 157, 282, 244, 232, 8, 128, 156, 239, 54, 255, 354, 211, 200, 190, 374, 361, 206, 272, 381, 214, 44, 98, 140, 25, 183, 37, 248, 380, 372, 217, 66, 169, 352, 176, 298, 148, 139, 185, 186, 216, 264, 6, 241, 311, 24, 114, 42, 73, 96, 60, 297, 92, 38, 256, 254, 318, 312, 159, 68, 197, 315, 292, 9, 266, 268, 167, 13, 83, 87, 238, 122, 65, 237, 378, 111, 97, 302, 99, 283, 182, 181, 129, 235, 135, 370, 45, 234, 276, 102, 304, 85, 125, 154, 175, 363, 219, 324, 101, 336, 337, 160, 325, 78, 161, 162, 127, 240, 265, 319, 205, 80, 147, 365, 86, 100, 295, 22, 286, 188, 351, 33, 17, 91, 213, 120, 339, 178, 221, 158, 314, 168, 251, 67, 364, 106, 126, 329, 10, 177, 299, 29, 309, 267, 63, 377, 199, 233, 23, 225, 19, 108, 220, 382, 345, 362, 342, 104, 166, 340, 366, 259, 48, 379, 142, 307, 227, 79, 349, 301, 41, 203, 121, 258, 130, 308, 116, 359, 333, 196, 131, 250, 231, 138, 201, 273, 152, 383, 11, 300, 193, 274, 218, 262, 103, 204, 71, 94, 253, 245, 88, 134, 280, 344, 367, 93, 89, 52, 164, 31, 132, 358, 270, 137, 61, 198, 153, 69, 224, 173, 210, 49, 20, 74, 202, 75, 34, 36, 229, 26, 151, 180, 27, 278, 334, 207, 47, 271, 1, 368, 110, 226, 172, 355, 15, 289, 189, 290, 76, 347, 109, 294, 287, 269, 228, 322, 321, 28, 149, 58, 105, 331, 170, 215, 118, 313, 323, 338, 356, 107, 260, 350, 145, 187, 279, 257, 376, 330, 284, 77, 348, 115, 310, 5, 222, 179, 119, 353, 40, 212, 184, 39, 53, 332, 81, 55, 82, 136, 375, 316, 21, 230, 263, 296, 243, 281, 150, 133, 317, 192, 35, 320, 327, 155, 285, 51, 18, 306, 144, 57, 64, 143, 371, 305, 346, 208, 223, 12, 70, 163, 242, 16, 30, 171, 43, 14, 123, 95, 249, 0, 141, 328, 46, 174, 277, 357, 291, 90, 59, 84, 124, 4, 117, 341, 2, 72, 194] + 48: [17, 41, 257, 24, 351, 16, 47, 278, 153, 258, 209, 250, 271, 354, 291, 111, 14, 40, 173, 335, 260, 336, 304, 308, 140, 139, 210, 329, 275, 367, 195, 29, 238, 333, 10, 196, 328, 5, 378, 321, 91, 211, 259, 373, 309, 355, 200, 227, 132, 6, 134, 192, 341, 180, 96, 178, 67, 225, 306, 114, 267, 287, 181, 179, 285, 70, 146, 216, 69, 148, 126, 129, 283, 203, 185, 371, 162, 218, 269, 169, 87, 149, 383, 334, 76, 74, 208, 175, 112, 202, 302, 39, 194, 60, 365, 314, 266, 121, 327, 293, 123, 231, 84, 292, 277, 154, 176, 245, 128, 343, 344, 319, 118, 79, 43, 206, 105, 244, 131, 1, 102, 236, 223, 155, 255, 360, 374, 346, 213, 350, 199, 30, 330, 44, 177, 232, 370, 168, 138, 151, 135, 110, 296, 136, 273, 234, 189, 305, 152, 357, 281, 127, 32, 37, 171, 220, 239, 19, 147, 380, 299, 221, 307, 253, 35, 143, 8, 23, 276, 99, 375, 252, 50, 212, 339, 63, 55, 294, 356, 348, 52, 226, 369, 184, 229, 89, 347, 12, 59, 163, 376, 167, 311, 116, 166, 115, 290, 31, 13, 81, 98, 61, 66, 103, 284, 215, 207, 316, 320, 170, 57, 36, 382, 3, 201, 337, 353, 358, 104, 186, 222, 340, 315, 18, 108, 93, 298, 54, 101, 109, 157, 325, 205, 160, 230, 352, 243, 326, 338, 68, 141, 150, 28, 34, 156, 73, 318, 349, 94, 165, 362, 122, 381, 301, 83, 240, 246, 120, 100, 49, 187, 310, 282, 183, 85, 279, 21, 235, 25, 233, 241, 204, 261, 313, 228, 265, 78, 145, 268, 359, 7, 247, 113, 72, 272, 92, 86, 248, 62, 117, 33, 48, 322, 174, 80, 119, 71, 217, 106, 46, 15, 65, 295, 363, 300, 0, 4, 53, 82, 214, 331, 251, 263, 77, 58, 264, 130, 324, 172, 280, 107, 27, 188, 364, 237, 270, 90, 342, 368, 242, 317, 97, 274, 224, 361, 197, 159, 26, 2, 372, 193, 190, 64, 158, 256, 366, 125, 124, 297, 249, 133, 38, 161, 75, 22, 289, 286, 95, 345, 332, 288, 20, 182, 254, 219, 88, 379, 11, 144, 323, 377, 262, 9, 51, 164, 142, 312, 45, 42, 198, 137, 56, 191, 303] + 49: [302, 135, 300, 322, 375, 38, 317, 337, 356, 64, 123, 191, 321, 227, 130, 22, 7, 266, 3, 347, 68, 188, 349, 92, 297, 344, 144, 362, 163, 295, 69, 2, 185, 236, 32, 370, 253, 60, 30, 336, 318, 139, 103, 106, 125, 363, 258, 373, 126, 325, 8, 254, 198, 55, 359, 46, 364, 372, 252, 24, 374, 262, 186, 250, 240, 20, 338, 61, 52, 167, 334, 26, 79, 44, 381, 366, 138, 376, 309, 0, 108, 87, 18, 129, 360, 39, 194, 328, 16, 224, 43, 339, 228, 343, 99, 288, 272, 275, 74, 195, 382, 296, 330, 14, 101, 104, 379, 369, 6, 201, 107, 5, 310, 41, 371, 71, 204, 324, 171, 247, 268, 270, 306, 131, 368, 323, 36, 244, 154, 51, 367, 50, 75, 212, 289, 94, 245, 34, 82, 358, 49, 182, 215, 263, 155, 105, 243, 333, 23, 128, 98, 345, 256, 147, 237, 219, 305, 78, 292, 100, 137, 4, 10, 280, 190, 168, 350, 203, 19, 66, 218, 264, 174, 170, 303, 119, 200, 232, 152, 291, 157, 353, 380, 175, 1, 365, 216, 142, 31, 326, 281, 383, 313, 116, 177, 45, 169, 277, 225, 351, 12, 354, 165, 11, 202, 213, 199, 255, 73, 196, 301, 156, 355, 145, 341, 230, 271, 293, 259, 331, 97, 140, 273, 307, 118, 299, 229, 217, 9, 115, 148, 329, 121, 28, 112, 173, 117, 151, 166, 231, 242, 181, 276, 269, 314, 86, 214, 134, 239, 113, 93, 159, 193, 158, 335, 286, 205, 279, 127, 179, 378, 206, 88, 265, 251, 150, 180, 91, 235, 80, 89, 327, 226, 124, 348, 249, 210, 136, 153, 260, 48, 340, 77, 238, 176, 283, 27, 287, 233, 234, 110, 132, 183, 311, 81, 308, 53, 192, 57, 162, 184, 59, 261, 220, 332, 70, 146, 178, 114, 361, 13, 42, 197, 65, 278, 120, 298, 56, 189, 33, 47, 285, 377, 102, 149, 209, 241, 312, 109, 221, 290, 62, 172, 143, 315, 96, 37, 257, 25, 187, 164, 54, 29, 67, 72, 282, 274, 141, 83, 304, 40, 246, 95, 223, 160, 267, 320, 284, 357, 207, 294, 15, 208, 133, 58, 63, 222, 352, 85, 84, 111, 21, 319, 346, 90, 76, 122, 17, 248, 35, 211, 342, 316, 161] + 50: [86, 116, 151, 28, 153, 330, 30, 95, 55, 230, 345, 365, 268, 154, 319, 294, 39, 123, 147, 83, 66, 222, 50, 93, 29, 0, 121, 244, 208, 378, 113, 34, 98, 173, 131, 241, 100, 312, 79, 195, 379, 357, 236, 212, 258, 31, 279, 374, 316, 146, 314, 375, 252, 105, 5, 308, 285, 207, 307, 257, 109, 373, 22, 117, 102, 255, 281, 344, 324, 176, 336, 264, 49, 125, 119, 6, 183, 85, 172, 288, 318, 104, 54, 111, 181, 110, 114, 363, 348, 382, 211, 289, 383, 60, 158, 136, 166, 273, 88, 133, 194, 371, 340, 4, 68, 89, 261, 364, 376, 35, 256, 132, 24, 250, 73, 185, 198, 323, 8, 120, 32, 218, 193, 298, 164, 94, 234, 178, 320, 112, 201, 44, 315, 262, 341, 353, 41, 269, 163, 243, 372, 179, 246, 84, 210, 361, 45, 16, 122, 310, 303, 223, 349, 286, 47, 144, 188, 370, 334, 271, 228, 189, 78, 127, 170, 346, 21, 161, 215, 233, 296, 187, 301, 267, 74, 356, 237, 150, 266, 182, 69, 352, 62, 155, 165, 359, 126, 191, 259, 40, 209, 283, 232, 284, 280, 184, 58, 219, 2, 347, 12, 293, 216, 309, 325, 367, 304, 343, 75, 168, 20, 202, 177, 77, 381, 332, 321, 38, 282, 253, 19, 134, 160, 292, 42, 291, 103, 167, 214, 14, 175, 51, 186, 43, 91, 369, 156, 36, 360, 23, 56, 333, 366, 331, 263, 206, 242, 337, 124, 82, 76, 221, 27, 97, 287, 295, 203, 290, 322, 204, 339, 11, 10, 225, 350, 229, 9, 224, 306, 169, 277, 17, 7, 171, 70, 227, 26, 67, 377, 130, 313, 139, 326, 251, 63, 138, 190, 260, 18, 140, 338, 272, 149, 53, 145, 362, 200, 162, 328, 302, 1, 137, 329, 270, 239, 247, 205, 297, 106, 87, 13, 317, 3, 311, 192, 213, 71, 342, 152, 37, 46, 354, 245, 52, 129, 299, 327, 249, 96, 197, 64, 300, 238, 90, 72, 115, 231, 226, 199, 335, 380, 174, 107, 143, 248, 148, 265, 118, 128, 235, 217, 159, 275, 180, 81, 278, 305, 65, 157, 254, 57, 25, 220, 15, 48, 240, 99, 135, 141, 355, 142, 368, 33, 80, 358, 274, 196, 276, 92, 108, 351, 101, 61, 59] + 51: [24, 219, 141, 157, 194, 336, 257, 266, 173, 202, 249, 261, 186, 81, 321, 100, 297, 15, 57, 150, 76, 121, 291, 326, 372, 314, 245, 209, 362, 258, 298, 68, 28, 22, 377, 151, 102, 95, 335, 271, 351, 361, 115, 319, 116, 132, 38, 21, 137, 60, 130, 183, 18, 119, 324, 30, 159, 154, 293, 348, 373, 104, 103, 285, 279, 79, 364, 80, 197, 174, 71, 290, 375, 270, 269, 292, 42, 59, 265, 288, 58, 201, 169, 156, 221, 152, 243, 85, 198, 248, 196, 315, 112, 272, 352, 226, 241, 67, 295, 299, 56, 34, 69, 54, 232, 14, 175, 2, 37, 64, 330, 259, 172, 10, 224, 307, 61, 277, 179, 138, 210, 176, 325, 8, 97, 33, 262, 223, 238, 88, 187, 139, 110, 122, 250, 247, 303, 192, 164, 27, 167, 7, 63, 155, 200, 356, 188, 5, 308, 310, 239, 338, 316, 158, 185, 327, 323, 127, 301, 92, 382, 280, 48, 365, 273, 1, 367, 255, 208, 180, 235, 26, 344, 190, 41, 334, 93, 148, 376, 267, 12, 228, 124, 263, 275, 181, 236, 349, 333, 214, 217, 231, 287, 182, 260, 134, 161, 222, 3, 312, 363, 66, 128, 36, 17, 52, 40, 11, 47, 366, 195, 332, 278, 43, 39, 318, 342, 284, 203, 49, 212, 329, 75, 94, 78, 378, 165, 302, 337, 193, 32, 346, 331, 360, 370, 289, 328, 53, 215, 29, 144, 113, 87, 253, 281, 142, 368, 311, 114, 339, 305, 274, 82, 123, 135, 246, 23, 254, 44, 234, 145, 340, 0, 70, 126, 357, 140, 168, 20, 359, 91, 171, 170, 240, 296, 117, 229, 99, 343, 282, 143, 237, 160, 317, 350, 25, 96, 131, 163, 16, 65, 153, 216, 107, 204, 84, 379, 149, 230, 256, 286, 111, 300, 304, 294, 162, 207, 313, 89, 125, 4, 225, 322, 98, 72, 146, 13, 345, 50, 306, 276, 353, 77, 355, 108, 206, 136, 177, 118, 341, 252, 369, 45, 184, 354, 244, 31, 178, 211, 129, 35, 380, 90, 320, 242, 55, 120, 133, 106, 6, 358, 283, 374, 86, 251, 383, 51, 218, 9, 147, 189, 101, 264, 105, 220, 74, 166, 205, 191, 109, 199, 227, 73, 371, 62, 233, 46, 268, 309, 213, 381, 19, 83, 347] + 52: [381, 45, 383, 310, 18, 203, 86, 264, 46, 38, 176, 180, 99, 39, 58, 51, 226, 36, 14, 272, 305, 247, 348, 375, 334, 223, 154, 308, 152, 278, 302, 338, 295, 360, 16, 175, 102, 127, 23, 319, 93, 194, 342, 147, 0, 376, 162, 212, 107, 64, 222, 248, 374, 242, 143, 126, 273, 240, 218, 370, 157, 60, 171, 221, 361, 213, 294, 303, 299, 11, 317, 106, 71, 33, 98, 359, 113, 324, 365, 356, 335, 12, 373, 204, 104, 62, 54, 74, 367, 22, 270, 139, 322, 158, 1, 94, 165, 346, 349, 256, 233, 115, 235, 189, 118, 363, 24, 164, 97, 172, 252, 285, 364, 351, 350, 207, 344, 132, 144, 199, 283, 90, 357, 312, 191, 300, 208, 148, 368, 232, 145, 254, 103, 174, 68, 95, 378, 29, 279, 352, 220, 111, 276, 241, 137, 321, 190, 32, 339, 160, 377, 225, 251, 197, 275, 142, 336, 271, 288, 87, 379, 167, 53, 268, 84, 141, 274, 366, 239, 195, 41, 266, 133, 30, 17, 161, 21, 380, 112, 122, 209, 290, 382, 355, 291, 281, 146, 27, 201, 198, 353, 159, 48, 323, 183, 210, 236, 124, 280, 109, 89, 100, 358, 202, 25, 289, 13, 128, 123, 5, 301, 287, 224, 136, 181, 261, 205, 214, 263, 196, 108, 286, 72, 69, 59, 230, 229, 329, 52, 85, 169, 56, 35, 105, 243, 40, 237, 28, 43, 19, 186, 73, 345, 269, 354, 315, 6, 91, 200, 101, 81, 65, 327, 362, 34, 219, 50, 178, 153, 66, 277, 156, 114, 265, 121, 250, 163, 44, 70, 341, 328, 333, 244, 150, 4, 185, 92, 246, 193, 259, 257, 227, 75, 119, 110, 249, 293, 187, 307, 151, 206, 168, 8, 173, 67, 125, 177, 347, 304, 255, 120, 63, 7, 10, 37, 83, 116, 326, 57, 129, 96, 314, 231, 130, 184, 245, 325, 166, 369, 253, 258, 188, 311, 42, 332, 182, 78, 260, 88, 117, 76, 170, 282, 179, 313, 79, 61, 31, 298, 140, 155, 234, 331, 316, 20, 318, 135, 297, 216, 267, 82, 80, 228, 262, 3, 55, 284, 49, 340, 15, 372, 371, 77, 26, 309, 296, 292, 337, 238, 47, 343, 131, 217, 320, 211, 192, 149, 215, 134, 306, 2, 330, 138, 9] + 53: [110, 18, 140, 153, 167, 312, 316, 185, 84, 114, 210, 272, 240, 231, 349, 196, 79, 337, 1, 190, 58, 377, 56, 90, 73, 319, 383, 89, 120, 117, 275, 310, 187, 191, 314, 252, 38, 103, 25, 176, 44, 19, 123, 289, 301, 192, 223, 265, 16, 162, 241, 77, 159, 105, 251, 331, 51, 212, 244, 22, 317, 302, 121, 168, 52, 102, 321, 101, 108, 163, 27, 344, 13, 92, 31, 219, 10, 76, 236, 133, 380, 17, 267, 307, 350, 42, 332, 325, 172, 67, 158, 220, 264, 147, 61, 71, 23, 227, 261, 5, 151, 146, 327, 232, 283, 193, 181, 99, 334, 155, 237, 127, 205, 126, 339, 255, 137, 366, 343, 245, 287, 179, 376, 95, 188, 72, 115, 333, 367, 34, 260, 290, 296, 379, 291, 171, 116, 15, 216, 136, 257, 39, 98, 318, 109, 224, 194, 253, 183, 281, 111, 28, 87, 82, 104, 11, 354, 345, 157, 24, 112, 47, 362, 371, 0, 234, 374, 263, 311, 363, 86, 20, 341, 149, 175, 370, 294, 305, 348, 124, 347, 214, 351, 326, 338, 150, 122, 278, 138, 9, 298, 304, 295, 273, 268, 269, 361, 259, 353, 30, 358, 249, 342, 355, 250, 365, 309, 284, 94, 49, 233, 166, 59, 14, 64, 356, 330, 46, 202, 107, 62, 37, 248, 2, 195, 154, 303, 165, 32, 322, 360, 130, 70, 300, 340, 375, 242, 80, 106, 161, 4, 292, 156, 144, 141, 378, 369, 169, 258, 7, 113, 148, 186, 280, 180, 271, 381, 78, 328, 57, 254, 246, 184, 211, 230, 142, 160, 119, 139, 239, 36, 178, 45, 21, 372, 213, 336, 173, 228, 50, 3, 197, 282, 96, 198, 29, 83, 286, 308, 164, 145, 225, 69, 335, 128, 206, 54, 201, 93, 135, 217, 320, 279, 221, 12, 152, 35, 174, 299, 100, 177, 33, 75, 129, 132, 26, 262, 235, 293, 207, 364, 199, 48, 288, 8, 238, 118, 368, 200, 277, 359, 306, 182, 43, 63, 204, 68, 247, 266, 134, 256, 313, 357, 189, 55, 53, 229, 41, 297, 270, 170, 125, 88, 373, 143, 276, 97, 352, 208, 382, 243, 66, 6, 285, 329, 324, 226, 65, 218, 323, 85, 274, 203, 222, 209, 81, 346, 60, 315, 215, 40, 131, 74, 91] + 54: [363, 13, 160, 205, 0, 36, 148, 232, 320, 336, 361, 378, 370, 383, 19, 200, 202, 358, 130, 212, 106, 123, 127, 354, 294, 125, 97, 326, 195, 229, 269, 292, 252, 193, 340, 344, 339, 156, 335, 239, 355, 293, 347, 243, 52, 270, 207, 78, 155, 10, 198, 94, 49, 298, 275, 74, 178, 77, 2, 342, 164, 352, 99, 225, 246, 61, 191, 145, 247, 138, 112, 349, 284, 57, 375, 50, 103, 105, 107, 33, 251, 89, 374, 297, 283, 290, 317, 113, 152, 365, 327, 18, 351, 46, 102, 188, 42, 174, 75, 197, 24, 184, 201, 62, 271, 108, 149, 222, 258, 53, 314, 196, 34, 55, 68, 244, 334, 343, 285, 241, 11, 8, 40, 76, 272, 368, 85, 151, 141, 362, 56, 256, 356, 172, 25, 41, 17, 1, 373, 9, 259, 206, 303, 37, 223, 20, 119, 143, 91, 177, 73, 175, 308, 261, 307, 126, 231, 98, 311, 132, 124, 381, 122, 167, 371, 359, 360, 165, 240, 250, 87, 332, 357, 382, 21, 136, 353, 12, 233, 51, 348, 30, 330, 27, 64, 54, 140, 242, 146, 15, 367, 291, 38, 133, 345, 101, 322, 182, 186, 137, 114, 238, 60, 203, 280, 214, 221, 58, 277, 142, 204, 304, 364, 110, 29, 65, 230, 369, 305, 181, 158, 92, 237, 22, 81, 63, 331, 104, 299, 80, 187, 121, 84, 217, 312, 111, 173, 139, 263, 199, 286, 260, 93, 26, 333, 321, 135, 366, 23, 219, 236, 210, 95, 82, 282, 180, 288, 323, 278, 144, 266, 224, 249, 4, 295, 150, 281, 16, 47, 45, 183, 48, 273, 234, 274, 28, 337, 257, 306, 372, 129, 159, 376, 209, 162, 194, 66, 226, 380, 216, 72, 329, 115, 220, 318, 69, 300, 346, 289, 350, 153, 267, 120, 309, 310, 325, 67, 171, 128, 6, 14, 379, 296, 287, 324, 86, 161, 109, 157, 313, 377, 245, 248, 227, 253, 39, 44, 117, 70, 265, 315, 163, 166, 192, 79, 35, 71, 90, 176, 3, 215, 279, 170, 179, 96, 88, 268, 116, 319, 83, 341, 228, 32, 185, 131, 301, 316, 235, 211, 134, 328, 147, 208, 118, 218, 190, 302, 255, 262, 213, 31, 154, 189, 338, 254, 7, 276, 5, 100, 264, 43, 59, 168, 169] + 55: [154, 254, 61, 201, 50, 369, 283, 186, 317, 215, 69, 305, 174, 27, 54, 63, 365, 261, 162, 45, 222, 245, 204, 34, 56, 301, 44, 5, 95, 286, 117, 367, 139, 324, 250, 96, 202, 227, 333, 233, 289, 322, 278, 97, 118, 75, 83, 285, 88, 307, 249, 312, 231, 257, 84, 219, 137, 181, 57, 109, 81, 111, 2, 366, 276, 79, 358, 160, 321, 248, 266, 196, 98, 238, 341, 216, 325, 240, 23, 212, 126, 338, 282, 32, 20, 16, 200, 300, 14, 360, 255, 1, 319, 129, 298, 351, 11, 252, 291, 221, 350, 42, 128, 237, 35, 38, 36, 100, 146, 311, 106, 213, 348, 47, 277, 264, 37, 66, 183, 205, 308, 39, 40, 164, 102, 343, 346, 31, 258, 145, 242, 9, 340, 165, 49, 368, 170, 361, 287, 153, 103, 275, 169, 167, 116, 339, 132, 138, 46, 246, 318, 82, 232, 354, 262, 158, 80, 89, 199, 290, 163, 378, 228, 22, 225, 147, 155, 108, 381, 182, 65, 349, 112, 105, 53, 294, 335, 58, 364, 229, 123, 357, 70, 272, 316, 265, 122, 85, 253, 209, 244, 230, 299, 370, 363, 12, 124, 217, 194, 296, 143, 107, 280, 25, 90, 218, 304, 60, 0, 380, 10, 171, 93, 336, 110, 345, 48, 62, 382, 71, 271, 236, 19, 157, 372, 315, 92, 24, 347, 7, 273, 99, 239, 371, 119, 207, 30, 297, 362, 59, 74, 234, 353, 114, 26, 224, 263, 52, 356, 168, 21, 142, 323, 4, 121, 293, 189, 91, 306, 198, 342, 120, 191, 131, 180, 309, 247, 134, 337, 375, 136, 87, 281, 177, 13, 310, 130, 151, 78, 223, 86, 377, 184, 193, 268, 55, 330, 195, 29, 355, 303, 8, 197, 284, 135, 314, 359, 243, 334, 288, 269, 235, 73, 3, 172, 190, 67, 302, 173, 125, 77, 292, 115, 256, 328, 179, 259, 274, 178, 214, 127, 374, 376, 379, 76, 270, 192, 331, 17, 352, 187, 313, 208, 150, 141, 148, 326, 43, 6, 320, 383, 251, 140, 94, 373, 161, 295, 18, 113, 327, 267, 279, 156, 210, 185, 101, 15, 104, 175, 220, 72, 28, 41, 176, 206, 329, 241, 33, 133, 64, 166, 144, 159, 152, 211, 226, 344, 260, 332, 149, 188, 68, 203, 51] + 56: [231, 142, 85, 224, 315, 146, 173, 202, 3, 90, 143, 208, 240, 65, 382, 217, 171, 318, 102, 229, 76, 109, 301, 74, 195, 324, 149, 286, 54, 375, 352, 203, 277, 237, 51, 61, 260, 197, 105, 280, 141, 201, 226, 93, 26, 366, 91, 235, 220, 77, 205, 135, 214, 368, 119, 37, 113, 108, 377, 165, 294, 327, 309, 182, 305, 248, 360, 232, 287, 234, 45, 247, 321, 306, 292, 101, 342, 198, 185, 62, 4, 43, 215, 194, 242, 104, 275, 168, 160, 206, 297, 96, 172, 16, 261, 365, 265, 363, 316, 223, 233, 12, 11, 145, 228, 29, 326, 380, 44, 263, 46, 344, 58, 268, 138, 314, 310, 67, 335, 25, 178, 9, 346, 246, 103, 269, 153, 262, 353, 78, 336, 111, 82, 156, 48, 238, 359, 22, 34, 190, 274, 24, 47, 216, 213, 371, 161, 154, 312, 136, 56, 271, 112, 350, 68, 114, 15, 322, 328, 80, 0, 92, 50, 42, 218, 289, 212, 174, 225, 351, 162, 188, 150, 285, 295, 209, 299, 200, 361, 358, 241, 267, 117, 19, 14, 278, 317, 151, 94, 28, 130, 325, 106, 303, 6, 71, 72, 73, 355, 133, 210, 124, 222, 256, 340, 158, 125, 302, 63, 137, 338, 227, 55, 290, 266, 7, 259, 184, 308, 140, 97, 27, 354, 127, 183, 293, 49, 349, 30, 296, 251, 39, 207, 40, 79, 270, 257, 307, 64, 370, 5, 98, 187, 99, 118, 69, 35, 164, 298, 323, 144, 52, 75, 163, 333, 41, 383, 376, 374, 243, 134, 347, 170, 59, 304, 88, 264, 284, 120, 334, 331, 1, 84, 341, 367, 32, 147, 155, 89, 66, 159, 128, 179, 33, 221, 276, 152, 131, 279, 53, 20, 83, 357, 193, 166, 13, 378, 57, 373, 272, 100, 186, 189, 329, 192, 38, 252, 282, 313, 139, 70, 239, 177, 116, 356, 332, 381, 253, 230, 17, 115, 319, 204, 157, 8, 107, 169, 18, 283, 281, 21, 364, 180, 132, 372, 36, 311, 273, 87, 2, 245, 250, 23, 255, 339, 291, 86, 176, 343, 81, 196, 362, 288, 10, 31, 123, 95, 330, 211, 258, 167, 199, 236, 219, 244, 148, 181, 129, 122, 379, 254, 337, 126, 191, 249, 320, 369, 300, 121, 110, 348, 175, 60, 345] + 57: [254, 56, 86, 234, 262, 275, 376, 293, 372, 301, 304, 343, 276, 167, 249, 12, 337, 175, 299, 375, 339, 280, 19, 285, 326, 127, 238, 214, 208, 351, 323, 307, 194, 218, 15, 233, 25, 174, 272, 228, 92, 71, 132, 122, 260, 239, 193, 143, 231, 6, 80, 243, 111, 210, 17, 303, 36, 67, 52, 367, 14, 124, 35, 176, 277, 55, 216, 363, 219, 78, 269, 202, 146, 359, 317, 99, 70, 47, 196, 379, 200, 361, 212, 34, 259, 108, 197, 91, 27, 203, 353, 18, 255, 222, 258, 201, 295, 223, 283, 58, 177, 120, 352, 41, 265, 48, 244, 229, 165, 257, 318, 338, 152, 38, 149, 334, 270, 242, 291, 105, 89, 128, 151, 347, 51, 289, 3, 29, 24, 256, 114, 69, 246, 101, 84, 144, 42, 284, 171, 162, 59, 374, 115, 77, 187, 65, 117, 319, 96, 358, 199, 371, 248, 192, 100, 181, 205, 235, 150, 182, 341, 147, 164, 186, 135, 66, 364, 221, 148, 366, 204, 354, 297, 183, 313, 215, 211, 263, 370, 327, 207, 104, 169, 251, 9, 54, 226, 311, 236, 383, 232, 294, 331, 140, 350, 250, 73, 116, 136, 98, 45, 50, 138, 340, 94, 198, 217, 62, 134, 102, 68, 330, 49, 87, 310, 245, 72, 7, 60, 75, 40, 377, 2, 362, 261, 333, 107, 278, 286, 156, 300, 163, 213, 184, 166, 28, 332, 224, 103, 189, 320, 63, 267, 123, 346, 349, 348, 81, 26, 126, 360, 37, 170, 368, 118, 44, 322, 21, 329, 23, 290, 188, 10, 292, 125, 57, 344, 88, 161, 298, 328, 369, 237, 4, 271, 336, 302, 85, 268, 79, 172, 113, 380, 178, 97, 168, 308, 131, 74, 220, 53, 306, 253, 382, 95, 190, 355, 109, 76, 82, 154, 121, 324, 309, 373, 46, 325, 155, 145, 357, 61, 30, 157, 314, 8, 160, 316, 312, 279, 282, 33, 32, 264, 129, 93, 133, 130, 64, 365, 356, 13, 209, 252, 0, 273, 153, 11, 158, 335, 345, 110, 173, 112, 321, 5, 378, 142, 195, 342, 16, 137, 185, 43, 83, 119, 296, 247, 241, 139, 227, 159, 381, 141, 22, 39, 191, 225, 90, 240, 281, 266, 206, 230, 31, 274, 180, 179, 20, 106, 288, 315, 1, 287, 305] + 58: [377, 280, 364, 201, 204, 340, 33, 38, 58, 134, 209, 308, 220, 71, 125, 319, 321, 335, 370, 10, 108, 150, 168, 184, 81, 144, 70, 170, 354, 349, 218, 233, 252, 292, 309, 3, 245, 104, 229, 244, 378, 161, 50, 221, 146, 137, 320, 279, 156, 191, 185, 381, 135, 253, 230, 268, 200, 216, 80, 147, 275, 196, 260, 236, 250, 347, 293, 100, 167, 301, 164, 158, 126, 45, 60, 48, 99, 37, 327, 173, 5, 57, 323, 23, 56, 376, 162, 192, 284, 339, 11, 113, 24, 109, 343, 165, 366, 36, 189, 73, 375, 300, 357, 180, 317, 51, 304, 166, 305, 116, 342, 106, 53, 382, 107, 179, 303, 290, 15, 313, 205, 257, 203, 198, 331, 241, 172, 64, 102, 117, 363, 212, 92, 72, 272, 328, 202, 302, 379, 338, 54, 337, 30, 291, 98, 289, 295, 255, 139, 350, 124, 278, 353, 163, 41, 17, 194, 69, 262, 111, 1, 181, 332, 148, 219, 128, 374, 127, 311, 22, 68, 132, 380, 258, 63, 13, 118, 213, 120, 91, 222, 9, 232, 130, 187, 25, 160, 334, 8, 264, 103, 52, 177, 49, 152, 315, 281, 62, 240, 288, 237, 330, 44, 153, 312, 235, 175, 97, 28, 251, 227, 318, 195, 276, 157, 78, 122, 299, 368, 6, 316, 238, 234, 345, 225, 359, 46, 159, 298, 206, 40, 85, 190, 129, 75, 154, 133, 261, 246, 361, 114, 93, 178, 141, 171, 55, 121, 176, 136, 296, 32, 188, 263, 322, 169, 367, 352, 266, 131, 186, 94, 243, 34, 267, 248, 373, 112, 47, 20, 215, 83, 228, 19, 138, 39, 365, 77, 101, 214, 31, 242, 59, 155, 21, 265, 142, 369, 89, 249, 271, 256, 18, 197, 82, 326, 2, 115, 360, 96, 74, 151, 14, 199, 341, 66, 239, 149, 231, 105, 294, 145, 174, 76, 211, 26, 324, 224, 314, 333, 371, 182, 29, 223, 217, 61, 210, 355, 16, 274, 325, 110, 208, 226, 95, 247, 310, 79, 344, 270, 119, 4, 306, 0, 362, 297, 372, 269, 140, 259, 348, 351, 12, 285, 383, 183, 65, 283, 282, 254, 88, 273, 27, 67, 277, 329, 143, 90, 43, 287, 87, 84, 193, 7, 356, 358, 42, 307, 207, 346, 123, 35, 86, 336, 286] + 59: [327, 51, 110, 93, 241, 56, 190, 308, 341, 215, 271, 189, 25, 223, 248, 69, 123, 246, 89, 230, 192, 375, 188, 154, 170, 175, 162, 258, 349, 149, 326, 11, 303, 351, 310, 79, 229, 262, 40, 373, 198, 61, 325, 261, 312, 335, 4, 268, 7, 178, 304, 252, 339, 2, 41, 145, 284, 270, 116, 235, 140, 265, 114, 62, 54, 101, 32, 207, 210, 305, 272, 90, 209, 172, 128, 297, 338, 289, 380, 343, 183, 301, 55, 111, 224, 221, 144, 35, 20, 180, 96, 129, 296, 239, 205, 117, 250, 294, 26, 57, 376, 269, 242, 34, 277, 358, 39, 15, 379, 371, 23, 161, 315, 120, 68, 135, 49, 103, 14, 139, 370, 263, 317, 109, 267, 214, 353, 299, 359, 3, 219, 293, 94, 204, 372, 112, 163, 356, 228, 48, 74, 44, 350, 77, 195, 247, 9, 133, 320, 222, 337, 71, 184, 357, 276, 347, 202, 298, 377, 290, 169, 47, 36, 227, 124, 148, 8, 345, 260, 374, 186, 342, 383, 179, 6, 0, 217, 118, 82, 99, 59, 311, 38, 45, 364, 158, 92, 64, 153, 88, 360, 171, 363, 381, 213, 176, 280, 10, 238, 63, 330, 24, 368, 323, 200, 251, 328, 91, 211, 236, 259, 43, 134, 191, 165, 72, 173, 70, 295, 278, 302, 355, 354, 193, 157, 275, 319, 122, 340, 95, 324, 212, 334, 329, 131, 194, 37, 336, 125, 244, 220, 5, 151, 164, 100, 286, 13, 206, 17, 249, 257, 83, 66, 233, 300, 86, 159, 306, 365, 333, 21, 106, 216, 52, 346, 226, 31, 196, 127, 245, 361, 27, 231, 119, 108, 104, 352, 147, 18, 255, 279, 19, 203, 138, 84, 321, 282, 87, 234, 197, 42, 58, 181, 142, 155, 287, 28, 132, 1, 264, 65, 97, 185, 309, 168, 29, 177, 156, 67, 115, 283, 167, 332, 313, 331, 243, 12, 143, 348, 107, 187, 307, 240, 121, 73, 256, 76, 281, 85, 22, 344, 362, 208, 102, 98, 378, 113, 46, 150, 266, 16, 218, 366, 314, 253, 53, 237, 78, 182, 75, 288, 232, 254, 174, 369, 199, 50, 166, 316, 126, 33, 285, 291, 382, 160, 60, 137, 292, 136, 201, 318, 152, 30, 105, 130, 80, 81, 225, 146, 274, 322, 273, 141, 367] + 60: [313, 317, 63, 154, 207, 348, 299, 288, 375, 22, 201, 355, 124, 218, 226, 242, 308, 89, 136, 210, 373, 62, 157, 214, 351, 262, 224, 135, 173, 4, 32, 101, 321, 10, 77, 87, 123, 8, 66, 380, 202, 50, 229, 163, 295, 352, 118, 109, 257, 140, 183, 274, 85, 79, 18, 258, 97, 164, 340, 44, 16, 235, 20, 162, 21, 95, 225, 198, 356, 71, 103, 73, 268, 234, 232, 336, 259, 304, 200, 219, 251, 379, 281, 275, 381, 76, 122, 152, 1, 116, 91, 144, 23, 374, 254, 177, 158, 2, 329, 27, 269, 110, 142, 30, 333, 320, 175, 227, 220, 153, 353, 231, 362, 188, 171, 100, 338, 178, 301, 289, 47, 312, 49, 117, 189, 344, 186, 190, 105, 45, 17, 292, 139, 120, 319, 272, 84, 11, 25, 322, 64, 19, 147, 138, 60, 247, 337, 184, 150, 240, 323, 370, 331, 137, 96, 367, 126, 230, 294, 131, 75, 318, 279, 293, 368, 67, 42, 68, 83, 167, 35, 245, 172, 6, 377, 70, 182, 325, 179, 111, 133, 194, 108, 206, 359, 51, 57, 233, 300, 363, 216, 298, 199, 78, 239, 278, 253, 217, 176, 346, 369, 383, 261, 291, 40, 48, 205, 15, 263, 36, 264, 31, 7, 168, 209, 72, 174, 222, 366, 213, 246, 132, 237, 273, 106, 371, 54, 378, 265, 212, 34, 98, 364, 221, 3, 107, 130, 148, 228, 86, 310, 316, 208, 311, 55, 37, 38, 360, 192, 315, 287, 159, 284, 339, 81, 166, 238, 114, 61, 53, 149, 270, 324, 165, 102, 113, 155, 193, 241, 93, 267, 187, 341, 125, 127, 43, 94, 39, 74, 354, 169, 255, 382, 285, 33, 376, 328, 191, 236, 372, 327, 309, 82, 358, 215, 52, 146, 307, 252, 249, 121, 347, 28, 345, 99, 283, 5, 90, 196, 282, 13, 306, 335, 244, 181, 119, 271, 248, 9, 134, 357, 197, 330, 326, 128, 343, 334, 290, 59, 58, 88, 56, 314, 12, 280, 29, 260, 303, 302, 112, 223, 69, 332, 26, 349, 14, 41, 160, 170, 129, 65, 180, 143, 277, 24, 305, 296, 286, 104, 204, 115, 266, 185, 365, 161, 151, 92, 141, 250, 156, 361, 350, 195, 243, 297, 46, 203, 0, 80, 342, 276, 256, 145, 211] + 61: [221, 44, 104, 37, 342, 246, 28, 217, 356, 155, 0, 58, 127, 366, 145, 306, 4, 54, 314, 226, 128, 321, 149, 319, 179, 159, 75, 308, 137, 174, 204, 262, 339, 376, 96, 94, 338, 286, 133, 260, 72, 365, 244, 354, 285, 175, 53, 208, 61, 294, 181, 43, 257, 186, 125, 359, 24, 265, 229, 171, 73, 68, 291, 87, 374, 259, 219, 274, 187, 60, 151, 287, 93, 380, 66, 32, 300, 147, 141, 144, 305, 373, 40, 116, 211, 1, 119, 227, 83, 97, 196, 176, 201, 95, 334, 343, 46, 264, 237, 309, 135, 52, 275, 124, 101, 256, 62, 189, 299, 184, 213, 169, 114, 328, 30, 71, 163, 272, 277, 350, 197, 209, 55, 243, 84, 245, 346, 317, 25, 123, 250, 183, 381, 59, 143, 156, 268, 23, 153, 81, 91, 99, 35, 210, 100, 241, 198, 282, 29, 206, 273, 132, 271, 234, 318, 382, 36, 270, 235, 160, 316, 368, 214, 26, 63, 18, 347, 134, 167, 19, 215, 15, 233, 69, 78, 295, 130, 301, 220, 355, 315, 10, 142, 31, 90, 3, 292, 340, 352, 364, 222, 39, 344, 203, 238, 349, 150, 336, 89, 375, 311, 255, 56, 254, 324, 117, 371, 177, 118, 113, 322, 341, 98, 232, 105, 41, 248, 178, 278, 112, 258, 304, 200, 240, 194, 13, 180, 357, 263, 323, 51, 192, 77, 166, 38, 8, 106, 312, 140, 267, 307, 122, 21, 188, 297, 27, 154, 280, 152, 49, 48, 320, 161, 205, 12, 57, 261, 2, 173, 22, 148, 251, 146, 367, 86, 165, 195, 42, 348, 252, 378, 79, 298, 7, 182, 266, 88, 351, 17, 129, 361, 379, 158, 80, 345, 310, 377, 289, 107, 247, 330, 168, 20, 313, 102, 269, 70, 358, 276, 281, 216, 326, 372, 327, 224, 190, 383, 9, 332, 185, 164, 296, 231, 170, 103, 115, 228, 47, 249, 283, 162, 74, 236, 50, 303, 157, 253, 369, 191, 11, 207, 67, 199, 293, 212, 16, 230, 120, 353, 34, 92, 109, 290, 85, 14, 111, 223, 76, 172, 360, 218, 110, 65, 82, 139, 242, 370, 6, 302, 279, 5, 108, 45, 288, 337, 329, 33, 126, 193, 239, 64, 131, 136, 335, 363, 333, 362, 121, 202, 225, 284, 331, 138, 325] +layer_updates_per_iter: 0 +num_slots: 384 diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-moe-load-balancer-configs/moe_load_balancer_gen_ep8_slots384.yaml b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-moe-load-balancer-configs/moe_load_balancer_gen_ep8_slots384.yaml new file mode 100644 index 0000000000..4a5c707042 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/dsv4-moe-load-balancer-configs/moe_load_balancer_gen_ep8_slots384.yaml @@ -0,0 +1,65 @@ +initial_global_assignments: + 0: [165, 45, 49, 259, 322, 235, 158, 163, 81, 29, 289, 70, 198, 144, 141, 181, 107, 43, 57, 361, 28, 248, 309, 262, 5, 91, 317, 137, 363, 192, 296, 270, 299, 326, 50, 252, 352, 316, 175, 2, 301, 125, 132, 304, 364, 273, 13, 373, 111, 279, 203, 150, 159, 348, 241, 295, 21, 80, 255, 376, 318, 272, 359, 72, 101, 233, 109, 129, 324, 208, 123, 314, 39, 133, 4, 108, 116, 216, 100, 379, 382, 263, 20, 210, 42, 212, 315, 334, 367, 251, 298, 329, 339, 8, 171, 274, 378, 135, 184, 0, 345, 177, 346, 188, 153, 242, 321, 65, 278, 209, 360, 284, 34, 313, 297, 311, 114, 44, 268, 234, 52, 308, 103, 146, 140, 19, 180, 320, 127, 14, 145, 350, 15, 64, 53, 69, 319, 164, 18, 226, 291, 88, 370, 185, 166, 254, 134, 217, 351, 155, 264, 332, 122, 62, 157, 68, 333, 280, 143, 205, 196, 283, 186, 200, 261, 148, 31, 358, 85, 187, 136, 240, 178, 99, 211, 195, 82, 328, 47, 102, 338, 340, 380, 179, 381, 250, 167, 374, 325, 323, 58, 201, 256, 225, 95, 330, 151, 383, 193, 182, 51, 104, 232, 63, 37, 126, 22, 307, 11, 341, 90, 258, 112, 300, 27, 89, 66, 32, 214, 249, 246, 275, 238, 292, 83, 93, 260, 229, 161, 253, 353, 152, 194, 357, 142, 24, 228, 130, 277, 61, 74, 371, 79, 290, 84, 119, 230, 169, 35, 202, 327, 139, 276, 349, 55, 33, 117, 156, 10, 36, 138, 7, 77, 375, 377, 172, 227, 331, 269, 366, 1, 25, 73, 67, 342, 98, 294, 206, 12, 219, 9, 285, 271, 121, 281, 168, 118, 247, 222, 245, 17, 173, 71, 86, 207, 312, 347, 30, 218, 176, 16, 3, 78, 243, 105, 183, 124, 174, 94, 337, 160, 199, 87, 244, 335, 365, 372, 369, 97, 106, 147, 237, 362, 286, 305, 115, 306, 191, 231, 220, 23, 189, 355, 288, 310, 303, 356, 236, 60, 224, 302, 162, 76, 190, 26, 204, 48, 343, 293, 282, 336, 54, 131, 56, 354, 265, 368, 170, 149, 38, 223, 257, 213, 46, 75, 59, 215, 221, 197, 267, 6, 154, 40, 96, 41, 113, 239, 266, 92, 287, 128, 344, 120, 110] + 1: [163, 177, 198, 292, 251, 18, 196, 378, 366, 328, 73, 153, 351, 195, 9, 32, 266, 285, 187, 261, 143, 282, 264, 202, 231, 189, 164, 238, 338, 326, 19, 33, 186, 51, 352, 346, 344, 77, 357, 220, 340, 217, 132, 16, 144, 28, 336, 265, 319, 169, 182, 225, 1, 80, 152, 40, 179, 41, 110, 271, 185, 273, 145, 173, 317, 355, 197, 194, 214, 176, 203, 239, 61, 98, 7, 305, 2, 20, 175, 136, 324, 11, 97, 8, 232, 314, 130, 246, 188, 373, 361, 45, 276, 339, 345, 15, 365, 72, 127, 82, 31, 106, 50, 21, 270, 123, 105, 248, 383, 243, 287, 71, 234, 212, 162, 379, 52, 53, 306, 376, 60, 205, 362, 204, 364, 17, 156, 68, 36, 284, 96, 181, 240, 103, 83, 297, 160, 207, 252, 343, 349, 341, 363, 327, 42, 91, 167, 178, 380, 118, 43, 241, 5, 75, 300, 244, 25, 46, 275, 86, 245, 301, 84, 92, 223, 116, 69, 360, 95, 76, 121, 259, 370, 107, 115, 62, 290, 26, 104, 113, 54, 119, 369, 331, 193, 209, 124, 155, 307, 354, 112, 368, 201, 382, 67, 272, 81, 44, 291, 329, 30, 210, 58, 56, 24, 59, 303, 247, 78, 274, 166, 142, 114, 93, 289, 278, 236, 298, 122, 330, 65, 286, 184, 154, 35, 230, 311, 268, 226, 333, 375, 372, 367, 221, 128, 322, 117, 257, 199, 255, 174, 277, 347, 134, 141, 308, 263, 131, 6, 325, 294, 299, 224, 320, 211, 260, 216, 99, 288, 109, 219, 228, 381, 342, 147, 237, 140, 170, 139, 318, 229, 165, 48, 293, 227, 55, 12, 49, 148, 309, 90, 151, 66, 374, 337, 157, 267, 100, 22, 158, 377, 111, 200, 126, 149, 254, 39, 0, 63, 250, 283, 29, 269, 242, 281, 74, 218, 280, 4, 47, 358, 171, 101, 102, 313, 222, 161, 332, 295, 321, 87, 315, 13, 302, 262, 356, 279, 371, 183, 253, 208, 64, 258, 304, 312, 215, 137, 180, 3, 120, 235, 85, 150, 168, 192, 70, 334, 23, 233, 138, 37, 89, 125, 146, 108, 335, 213, 206, 88, 38, 133, 256, 79, 348, 249, 14, 296, 57, 172, 350, 94, 323, 353, 27, 190, 129, 316, 135, 10, 191, 159, 359, 310, 34] + 2: [143, 295, 242, 180, 96, 137, 321, 78, 111, 173, 73, 245, 133, 325, 217, 235, 303, 165, 344, 273, 68, 67, 21, 138, 46, 171, 193, 129, 372, 252, 365, 219, 239, 47, 178, 261, 302, 288, 8, 164, 310, 37, 353, 88, 287, 337, 40, 374, 248, 3, 212, 230, 225, 28, 197, 33, 101, 284, 56, 70, 172, 368, 63, 333, 159, 209, 17, 87, 293, 92, 383, 148, 253, 265, 181, 103, 38, 312, 176, 179, 77, 238, 376, 319, 54, 71, 251, 98, 117, 227, 240, 289, 301, 202, 363, 161, 53, 223, 290, 314, 14, 5, 189, 213, 84, 166, 286, 44, 123, 45, 200, 82, 285, 61, 206, 224, 27, 262, 120, 218, 41, 65, 309, 69, 256, 12, 257, 0, 25, 281, 141, 352, 329, 99, 335, 158, 378, 283, 249, 315, 358, 282, 255, 366, 43, 115, 62, 86, 343, 298, 125, 162, 110, 322, 280, 278, 31, 51, 274, 24, 311, 79, 299, 263, 50, 203, 201, 76, 332, 241, 169, 250, 106, 39, 268, 2, 36, 334, 264, 359, 216, 370, 338, 307, 382, 177, 369, 64, 228, 354, 300, 381, 132, 93, 232, 139, 357, 259, 100, 192, 155, 124, 355, 271, 220, 22, 328, 85, 6, 342, 7, 90, 269, 210, 361, 297, 349, 308, 186, 254, 136, 49, 318, 23, 195, 346, 331, 57, 243, 371, 114, 267, 336, 222, 13, 380, 32, 339, 199, 330, 112, 194, 168, 304, 341, 221, 247, 60, 153, 97, 157, 135, 105, 296, 279, 340, 196, 174, 170, 347, 55, 188, 58, 128, 360, 145, 348, 276, 364, 121, 29, 362, 134, 4, 272, 231, 292, 373, 167, 59, 72, 234, 260, 127, 119, 270, 190, 379, 108, 89, 91, 377, 102, 1, 151, 149, 109, 191, 66, 118, 42, 26, 83, 244, 323, 204, 130, 356, 126, 327, 187, 198, 163, 16, 15, 160, 154, 142, 104, 94, 275, 35, 52, 144, 226, 95, 183, 48, 113, 345, 214, 266, 316, 208, 375, 258, 313, 215, 19, 122, 351, 81, 34, 75, 185, 350, 131, 306, 184, 324, 146, 11, 30, 294, 140, 18, 156, 320, 20, 236, 277, 10, 175, 116, 9, 305, 150, 152, 211, 367, 207, 233, 291, 107, 205, 74, 182, 147, 317, 80, 326, 237, 246, 229] + 3: [66, 171, 13, 62, 252, 331, 201, 120, 58, 231, 74, 316, 12, 87, 78, 288, 203, 374, 375, 299, 344, 197, 266, 280, 144, 366, 96, 249, 128, 7, 356, 321, 86, 133, 379, 140, 121, 329, 1, 95, 175, 60, 226, 233, 324, 47, 222, 370, 34, 14, 131, 80, 158, 180, 139, 178, 164, 37, 287, 174, 25, 118, 279, 142, 32, 111, 300, 274, 256, 57, 65, 246, 30, 160, 340, 70, 67, 15, 85, 346, 254, 206, 116, 220, 191, 45, 253, 352, 315, 69, 341, 257, 198, 311, 38, 301, 310, 27, 3, 275, 19, 192, 24, 150, 6, 102, 336, 2, 212, 239, 219, 240, 382, 161, 289, 262, 176, 345, 217, 53, 270, 117, 255, 125, 196, 115, 184, 232, 91, 132, 72, 101, 93, 179, 372, 242, 159, 54, 148, 320, 244, 5, 17, 278, 260, 18, 71, 314, 225, 333, 151, 50, 213, 63, 342, 358, 138, 221, 36, 290, 136, 109, 295, 277, 92, 98, 103, 55, 327, 135, 48, 328, 100, 353, 188, 305, 165, 381, 362, 293, 258, 183, 16, 210, 360, 90, 330, 349, 347, 371, 247, 292, 377, 40, 313, 230, 31, 97, 348, 49, 337, 339, 343, 211, 189, 304, 119, 21, 264, 8, 359, 312, 114, 41, 218, 235, 112, 105, 76, 214, 276, 286, 20, 281, 106, 259, 110, 269, 285, 245, 251, 338, 369, 147, 75, 124, 152, 33, 332, 303, 365, 302, 163, 46, 317, 261, 157, 81, 42, 238, 207, 10, 267, 296, 182, 223, 357, 162, 154, 149, 243, 126, 224, 380, 355, 129, 268, 82, 237, 322, 361, 291, 83, 61, 68, 309, 190, 0, 323, 107, 284, 368, 89, 265, 173, 167, 335, 59, 319, 79, 263, 209, 51, 208, 363, 334, 137, 26, 168, 227, 325, 195, 177, 29, 271, 364, 99, 351, 297, 204, 108, 236, 169, 273, 44, 104, 306, 248, 187, 294, 376, 130, 228, 153, 354, 378, 73, 156, 141, 205, 127, 308, 326, 185, 52, 199, 39, 22, 194, 35, 350, 56, 318, 145, 215, 186, 170, 250, 122, 77, 11, 64, 202, 84, 172, 229, 155, 181, 200, 166, 94, 307, 113, 383, 43, 216, 298, 367, 123, 373, 282, 28, 283, 134, 88, 234, 143, 241, 272, 23, 4, 193, 9, 146] + 4: [343, 148, 375, 259, 228, 257, 302, 345, 274, 76, 200, 361, 44, 312, 251, 35, 278, 276, 290, 133, 62, 379, 261, 64, 313, 94, 284, 39, 306, 121, 26, 180, 304, 201, 355, 96, 115, 353, 55, 368, 190, 285, 154, 2, 242, 120, 182, 277, 331, 142, 37, 24, 239, 105, 143, 253, 155, 41, 275, 357, 160, 8, 198, 187, 38, 335, 264, 189, 330, 169, 149, 229, 70, 240, 370, 220, 349, 303, 175, 249, 334, 40, 286, 126, 97, 163, 162, 366, 224, 301, 174, 20, 67, 336, 332, 351, 3, 319, 254, 12, 219, 88, 5, 136, 47, 329, 113, 77, 364, 101, 333, 22, 110, 283, 244, 0, 166, 72, 185, 341, 216, 232, 158, 288, 66, 52, 346, 122, 170, 118, 71, 213, 311, 203, 316, 247, 30, 245, 363, 227, 119, 272, 58, 282, 271, 226, 152, 124, 350, 172, 45, 241, 13, 197, 139, 318, 196, 27, 28, 280, 369, 179, 324, 243, 53, 356, 297, 323, 46, 262, 354, 84, 138, 267, 78, 103, 291, 147, 111, 255, 205, 378, 362, 191, 238, 293, 326, 337, 192, 321, 63, 305, 315, 95, 117, 18, 300, 16, 376, 308, 36, 377, 325, 61, 372, 57, 83, 217, 73, 260, 207, 358, 42, 168, 360, 352, 132, 269, 383, 25, 310, 92, 50, 230, 29, 327, 211, 223, 10, 186, 374, 279, 173, 134, 248, 89, 382, 17, 79, 90, 214, 15, 273, 340, 258, 193, 19, 6, 86, 59, 167, 246, 151, 235, 43, 49, 7, 82, 141, 183, 116, 307, 98, 93, 14, 108, 69, 100, 137, 292, 176, 234, 157, 140, 210, 31, 127, 215, 48, 112, 91, 202, 11, 212, 195, 225, 338, 107, 209, 23, 194, 34, 222, 320, 281, 171, 348, 56, 347, 4, 32, 314, 218, 344, 161, 177, 159, 99, 85, 65, 114, 359, 75, 125, 80, 135, 367, 309, 263, 184, 51, 181, 237, 109, 130, 81, 129, 380, 1, 265, 373, 236, 270, 208, 231, 104, 296, 371, 87, 188, 199, 250, 9, 150, 165, 342, 298, 145, 106, 365, 339, 204, 68, 317, 153, 54, 221, 252, 256, 33, 60, 74, 131, 322, 328, 178, 294, 206, 295, 146, 289, 268, 156, 381, 21, 123, 164, 144, 102, 266, 299, 233, 128, 287] + 5: [61, 305, 231, 141, 1, 223, 365, 64, 132, 200, 91, 182, 304, 70, 268, 238, 319, 24, 34, 220, 295, 186, 312, 19, 106, 250, 161, 90, 334, 236, 162, 55, 283, 378, 244, 58, 373, 111, 307, 282, 158, 294, 112, 348, 167, 57, 196, 302, 213, 105, 272, 154, 98, 265, 381, 41, 351, 7, 192, 74, 233, 313, 163, 264, 38, 207, 258, 377, 35, 144, 274, 227, 366, 300, 219, 289, 237, 360, 94, 376, 221, 263, 135, 92, 125, 101, 126, 85, 146, 42, 339, 142, 225, 50, 118, 354, 128, 100, 6, 165, 260, 49, 30, 379, 279, 88, 247, 22, 173, 137, 222, 174, 206, 3, 320, 315, 228, 79, 242, 183, 353, 240, 150, 325, 330, 13, 77, 62, 113, 129, 68, 316, 189, 290, 120, 73, 36, 123, 310, 176, 278, 286, 127, 75, 214, 276, 288, 208, 160, 60, 169, 78, 215, 352, 89, 335, 371, 155, 37, 369, 67, 143, 314, 218, 104, 115, 216, 39, 343, 121, 257, 203, 275, 149, 29, 191, 235, 71, 270, 298, 297, 347, 185, 309, 328, 357, 273, 10, 367, 157, 321, 177, 355, 346, 52, 172, 5, 178, 188, 102, 145, 43, 147, 25, 31, 8, 375, 2, 69, 204, 26, 45, 301, 99, 12, 253, 14, 239, 138, 136, 119, 9, 344, 232, 359, 103, 107, 168, 322, 245, 292, 267, 97, 336, 180, 27, 110, 82, 179, 303, 190, 217, 287, 259, 256, 65, 193, 211, 130, 17, 246, 338, 327, 66, 15, 358, 108, 318, 175, 333, 16, 195, 266, 329, 20, 382, 63, 284, 281, 80, 291, 271, 374, 299, 199, 40, 148, 32, 53, 285, 243, 340, 109, 116, 280, 96, 277, 368, 249, 194, 356, 21, 308, 81, 370, 372, 72, 212, 331, 364, 124, 131, 171, 59, 181, 140, 210, 153, 54, 28, 51, 345, 230, 362, 341, 76, 383, 87, 134, 229, 363, 114, 18, 156, 224, 95, 11, 139, 350, 234, 117, 248, 83, 164, 337, 44, 84, 296, 269, 159, 306, 47, 262, 122, 255, 252, 293, 198, 209, 4, 151, 332, 184, 251, 324, 152, 33, 46, 380, 93, 56, 201, 241, 317, 254, 48, 361, 349, 205, 323, 342, 187, 261, 86, 133, 197, 311, 202, 0, 226, 166, 326, 23, 170] + 6: [97, 281, 212, 306, 46, 371, 36, 318, 119, 246, 34, 239, 0, 113, 352, 236, 223, 202, 44, 211, 98, 280, 207, 116, 197, 261, 148, 199, 124, 349, 381, 366, 285, 152, 328, 117, 303, 183, 302, 29, 91, 344, 61, 65, 167, 35, 40, 90, 142, 240, 297, 249, 11, 67, 378, 216, 293, 84, 208, 129, 151, 176, 377, 278, 304, 111, 383, 219, 136, 365, 237, 242, 86, 217, 138, 81, 198, 224, 54, 309, 235, 130, 360, 162, 338, 313, 339, 92, 19, 93, 228, 153, 214, 320, 161, 164, 308, 253, 146, 27, 229, 31, 375, 181, 100, 112, 287, 289, 234, 165, 369, 336, 101, 190, 139, 327, 10, 157, 22, 110, 284, 23, 194, 316, 83, 105, 96, 16, 379, 357, 74, 337, 256, 348, 89, 356, 140, 127, 172, 353, 252, 363, 325, 247, 322, 258, 177, 160, 55, 120, 173, 163, 88, 368, 196, 184, 300, 301, 109, 245, 317, 37, 321, 77, 15, 154, 210, 69, 159, 243, 254, 329, 333, 346, 215, 42, 134, 292, 62, 108, 266, 351, 222, 250, 78, 141, 273, 200, 75, 149, 71, 323, 76, 174, 166, 103, 341, 205, 350, 359, 232, 189, 118, 291, 345, 277, 185, 380, 231, 279, 133, 64, 264, 58, 6, 104, 276, 82, 362, 122, 168, 358, 8, 70, 310, 312, 203, 288, 102, 294, 131, 201, 238, 227, 340, 218, 132, 135, 204, 257, 274, 343, 66, 295, 269, 180, 45, 307, 187, 57, 188, 178, 259, 95, 179, 41, 311, 158, 221, 107, 332, 286, 271, 87, 225, 330, 355, 145, 18, 193, 298, 268, 364, 244, 80, 347, 290, 51, 21, 374, 171, 241, 1, 20, 72, 25, 275, 342, 376, 53, 99, 59, 267, 209, 9, 260, 94, 56, 255, 79, 370, 270, 128, 182, 49, 12, 282, 144, 324, 220, 262, 206, 63, 335, 372, 126, 48, 3, 334, 299, 73, 123, 106, 226, 85, 24, 47, 38, 7, 192, 382, 2, 265, 33, 315, 373, 32, 305, 233, 5, 28, 326, 175, 13, 367, 283, 319, 155, 156, 354, 115, 230, 68, 30, 169, 114, 143, 296, 361, 43, 137, 60, 4, 14, 248, 50, 147, 272, 125, 39, 150, 251, 17, 195, 121, 263, 26, 213, 170, 191, 314, 52, 186, 331] + 7: [327, 265, 25, 272, 133, 271, 378, 328, 27, 73, 164, 248, 349, 288, 325, 208, 219, 191, 264, 9, 311, 204, 32, 178, 335, 121, 353, 373, 146, 68, 153, 79, 151, 287, 295, 1, 312, 3, 63, 198, 346, 12, 347, 154, 279, 110, 202, 364, 7, 229, 201, 157, 239, 46, 99, 329, 61, 281, 313, 267, 4, 31, 360, 284, 255, 62, 300, 214, 189, 245, 237, 107, 200, 86, 226, 323, 196, 291, 93, 203, 269, 106, 112, 72, 123, 39, 381, 159, 95, 66, 165, 138, 376, 236, 135, 257, 274, 142, 103, 162, 361, 342, 118, 212, 370, 302, 294, 82, 262, 23, 148, 235, 210, 377, 344, 167, 368, 109, 130, 5, 83, 44, 134, 301, 40, 261, 101, 351, 318, 166, 169, 218, 254, 379, 124, 315, 359, 150, 172, 127, 355, 30, 160, 50, 49, 375, 340, 369, 108, 28, 71, 188, 0, 94, 324, 100, 217, 119, 84, 250, 70, 170, 213, 132, 206, 26, 136, 190, 293, 102, 152, 310, 161, 8, 22, 345, 126, 320, 363, 69, 85, 111, 252, 228, 242, 304, 197, 143, 34, 326, 371, 42, 292, 41, 18, 341, 258, 187, 13, 97, 244, 131, 183, 47, 289, 285, 2, 319, 356, 21, 90, 241, 234, 140, 20, 180, 211, 58, 51, 53, 232, 227, 87, 52, 366, 35, 209, 194, 88, 173, 297, 334, 246, 45, 205, 54, 282, 240, 365, 60, 175, 64, 105, 163, 336, 176, 139, 15, 174, 181, 14, 215, 278, 17, 270, 296, 81, 224, 330, 333, 259, 16, 92, 19, 220, 380, 141, 56, 251, 309, 38, 193, 184, 216, 286, 316, 10, 367, 277, 78, 266, 57, 147, 331, 156, 223, 348, 171, 182, 263, 382, 298, 308, 260, 253, 48, 104, 114, 11, 113, 74, 168, 337, 374, 383, 177, 158, 98, 321, 89, 358, 129, 322, 6, 305, 306, 283, 247, 149, 120, 37, 222, 33, 207, 96, 231, 317, 128, 195, 230, 290, 144, 122, 179, 91, 357, 275, 192, 343, 362, 238, 350, 280, 354, 77, 256, 249, 307, 125, 29, 55, 339, 117, 221, 372, 243, 24, 314, 185, 186, 137, 273, 67, 155, 233, 65, 116, 225, 115, 268, 75, 36, 59, 80, 43, 303, 299, 332, 352, 199, 76, 145, 276, 338] + 8: [360, 308, 273, 238, 224, 272, 36, 252, 316, 364, 131, 128, 75, 109, 119, 330, 165, 219, 198, 113, 233, 52, 114, 101, 263, 379, 191, 90, 310, 136, 327, 200, 325, 236, 93, 323, 351, 324, 129, 340, 123, 305, 139, 344, 57, 262, 58, 234, 253, 162, 371, 279, 106, 336, 228, 276, 157, 294, 97, 311, 69, 23, 226, 179, 17, 303, 167, 196, 177, 125, 8, 284, 383, 25, 116, 249, 32, 382, 143, 140, 35, 260, 322, 134, 153, 15, 271, 26, 268, 80, 138, 288, 242, 127, 96, 170, 290, 362, 152, 214, 338, 12, 88, 81, 251, 209, 377, 374, 346, 83, 34, 240, 261, 178, 137, 50, 334, 211, 91, 61, 206, 98, 73, 293, 110, 266, 16, 199, 89, 295, 375, 365, 33, 146, 270, 269, 160, 355, 274, 301, 49, 163, 1, 207, 359, 201, 353, 104, 292, 13, 381, 74, 186, 349, 168, 148, 181, 188, 333, 258, 78, 275, 368, 183, 300, 141, 332, 60, 45, 117, 174, 71, 297, 47, 37, 126, 99, 339, 306, 14, 347, 24, 76, 158, 77, 245, 43, 203, 345, 176, 55, 102, 5, 239, 221, 281, 218, 108, 243, 337, 20, 255, 328, 280, 166, 22, 27, 283, 202, 169, 304, 194, 320, 192, 2, 358, 335, 217, 28, 232, 350, 299, 41, 155, 105, 85, 159, 369, 287, 357, 220, 244, 208, 285, 51, 53, 124, 0, 227, 132, 67, 264, 321, 204, 3, 302, 241, 64, 122, 380, 11, 86, 225, 172, 356, 147, 92, 65, 215, 112, 317, 120, 318, 4, 42, 254, 87, 247, 151, 373, 372, 313, 144, 235, 205, 185, 46, 277, 84, 348, 180, 354, 142, 331, 222, 210, 296, 100, 31, 367, 10, 182, 246, 161, 173, 282, 291, 95, 18, 115, 39, 107, 133, 6, 231, 121, 278, 309, 130, 48, 197, 66, 59, 212, 237, 257, 184, 149, 190, 314, 79, 265, 145, 29, 54, 326, 171, 103, 319, 70, 343, 342, 256, 361, 19, 376, 135, 21, 62, 229, 250, 7, 312, 154, 30, 223, 44, 156, 195, 366, 150, 38, 298, 216, 40, 189, 267, 164, 341, 72, 187, 230, 118, 248, 352, 259, 307, 68, 56, 378, 9, 175, 315, 111, 193, 370, 329, 286, 363, 94, 82, 289, 63, 213] + 9: [92, 244, 81, 372, 353, 184, 15, 138, 210, 67, 239, 258, 161, 135, 120, 297, 231, 192, 116, 186, 4, 230, 238, 237, 343, 235, 248, 382, 197, 261, 112, 291, 71, 298, 38, 229, 86, 34, 383, 266, 279, 44, 204, 157, 61, 236, 234, 26, 304, 64, 166, 280, 357, 179, 272, 217, 281, 136, 177, 9, 82, 377, 137, 88, 180, 314, 259, 35, 105, 36, 66, 141, 39, 324, 109, 307, 275, 233, 254, 110, 375, 310, 162, 347, 170, 193, 58, 240, 328, 284, 85, 164, 182, 77, 338, 57, 255, 313, 95, 143, 260, 219, 381, 83, 242, 271, 14, 342, 312, 360, 302, 289, 228, 6, 97, 99, 195, 123, 150, 218, 359, 104, 13, 354, 60, 45, 300, 2, 320, 251, 352, 232, 227, 69, 337, 140, 325, 73, 101, 378, 250, 27, 53, 340, 349, 185, 103, 331, 215, 350, 351, 1, 370, 214, 23, 309, 133, 30, 356, 292, 306, 247, 107, 129, 256, 336, 11, 332, 5, 308, 246, 8, 100, 148, 174, 358, 253, 305, 134, 89, 346, 167, 290, 368, 345, 249, 93, 200, 28, 122, 152, 348, 327, 47, 262, 212, 315, 252, 222, 344, 376, 287, 121, 56, 98, 317, 363, 268, 264, 0, 183, 41, 269, 139, 16, 189, 106, 75, 208, 50, 52, 245, 190, 203, 371, 330, 76, 211, 48, 188, 293, 243, 176, 118, 149, 273, 91, 265, 369, 365, 355, 295, 49, 78, 196, 299, 155, 198, 127, 329, 209, 32, 131, 285, 187, 3, 277, 54, 364, 20, 142, 267, 70, 181, 274, 80, 339, 65, 130, 117, 373, 114, 42, 286, 333, 21, 221, 241, 326, 68, 108, 361, 278, 194, 362, 25, 55, 173, 31, 263, 316, 10, 220, 19, 74, 29, 201, 159, 51, 132, 22, 367, 158, 154, 33, 334, 321, 311, 163, 323, 124, 171, 288, 319, 17, 18, 379, 46, 62, 322, 84, 301, 270, 125, 199, 59, 374, 43, 225, 63, 223, 113, 40, 7, 24, 168, 226, 94, 90, 147, 96, 172, 165, 37, 146, 380, 341, 282, 102, 151, 144, 153, 169, 318, 128, 87, 145, 207, 206, 111, 72, 202, 276, 175, 303, 205, 213, 79, 115, 12, 224, 216, 178, 335, 160, 191, 366, 119, 296, 294, 257, 156, 126, 283] + 10: [319, 310, 137, 325, 309, 73, 1, 263, 241, 196, 308, 91, 293, 259, 342, 360, 212, 223, 145, 90, 321, 21, 81, 329, 20, 338, 383, 36, 247, 193, 123, 343, 307, 350, 100, 173, 303, 168, 165, 334, 157, 344, 63, 14, 89, 253, 68, 369, 5, 146, 268, 289, 331, 102, 134, 179, 351, 330, 211, 67, 272, 363, 374, 148, 306, 149, 155, 103, 214, 288, 311, 339, 256, 37, 230, 162, 12, 174, 239, 185, 282, 96, 159, 222, 355, 4, 158, 94, 273, 30, 84, 277, 160, 166, 358, 336, 300, 164, 192, 48, 167, 225, 170, 161, 40, 182, 201, 213, 234, 368, 186, 208, 249, 285, 250, 0, 72, 296, 219, 9, 197, 41, 51, 71, 184, 317, 50, 183, 65, 299, 80, 187, 373, 35, 200, 118, 172, 352, 152, 52, 238, 176, 295, 283, 320, 121, 301, 181, 337, 2, 144, 54, 153, 210, 304, 58, 353, 252, 356, 204, 189, 280, 266, 233, 46, 3, 122, 335, 26, 346, 15, 258, 64, 275, 323, 261, 226, 364, 139, 206, 86, 17, 6, 34, 83, 207, 156, 242, 110, 236, 29, 297, 365, 24, 31, 294, 108, 279, 79, 195, 178, 114, 177, 154, 115, 130, 240, 377, 190, 376, 382, 132, 33, 129, 232, 85, 27, 357, 379, 25, 109, 318, 45, 313, 298, 113, 209, 62, 372, 119, 217, 16, 203, 281, 333, 202, 270, 381, 70, 366, 194, 265, 18, 246, 221, 126, 101, 22, 78, 66, 237, 269, 7, 243, 345, 274, 98, 38, 218, 128, 198, 107, 105, 106, 314, 112, 74, 97, 278, 354, 140, 362, 370, 244, 169, 302, 111, 147, 224, 133, 276, 44, 347, 227, 28, 43, 23, 324, 264, 151, 216, 93, 141, 367, 57, 215, 171, 138, 99, 248, 380, 312, 348, 284, 77, 231, 61, 32, 116, 87, 56, 75, 315, 378, 327, 267, 340, 39, 254, 163, 175, 220, 131, 287, 127, 305, 322, 245, 53, 104, 191, 143, 10, 188, 82, 349, 55, 180, 286, 255, 262, 69, 92, 8, 117, 19, 124, 47, 76, 11, 136, 49, 316, 271, 359, 199, 229, 135, 341, 60, 142, 59, 328, 371, 42, 125, 332, 95, 13, 257, 326, 375, 235, 205, 292, 251, 88, 120, 361, 150, 260, 228, 291, 290] + 11: [130, 221, 126, 299, 149, 190, 29, 218, 192, 141, 39, 20, 13, 250, 283, 107, 348, 92, 195, 86, 341, 255, 101, 327, 63, 30, 338, 179, 125, 264, 113, 62, 143, 184, 360, 222, 240, 132, 380, 229, 180, 181, 202, 123, 220, 321, 134, 243, 314, 235, 100, 77, 363, 145, 102, 22, 247, 330, 95, 381, 207, 320, 161, 45, 249, 226, 315, 340, 316, 14, 208, 301, 219, 59, 353, 193, 234, 383, 142, 263, 230, 271, 266, 112, 310, 167, 68, 183, 60, 53, 28, 239, 223, 25, 109, 12, 296, 150, 248, 156, 65, 119, 300, 236, 217, 343, 286, 318, 17, 297, 47, 280, 2, 373, 172, 151, 89, 298, 186, 270, 211, 168, 354, 349, 291, 293, 174, 182, 46, 194, 375, 210, 21, 214, 278, 277, 175, 369, 189, 276, 281, 72, 191, 335, 163, 85, 44, 252, 3, 99, 325, 345, 351, 284, 73, 177, 154, 313, 6, 124, 127, 159, 166, 374, 203, 96, 254, 242, 294, 245, 359, 91, 164, 57, 129, 188, 309, 274, 103, 106, 4, 140, 304, 328, 216, 139, 51, 206, 334, 50, 308, 269, 110, 215, 116, 260, 88, 233, 267, 332, 331, 128, 204, 282, 136, 23, 319, 170, 262, 75, 347, 10, 224, 289, 41, 81, 336, 279, 364, 302, 187, 64, 43, 7, 74, 231, 5, 323, 201, 322, 305, 379, 344, 246, 253, 76, 212, 137, 307, 241, 98, 83, 11, 357, 40, 326, 199, 24, 152, 165, 131, 228, 205, 303, 94, 367, 69, 117, 176, 56, 49, 197, 259, 35, 147, 292, 378, 158, 118, 115, 48, 18, 138, 358, 37, 84, 288, 290, 71, 342, 361, 352, 111, 82, 153, 256, 32, 42, 377, 90, 121, 15, 227, 38, 162, 0, 54, 178, 244, 146, 169, 196, 346, 213, 370, 80, 26, 251, 133, 365, 58, 209, 362, 366, 34, 135, 324, 79, 258, 355, 237, 382, 108, 306, 173, 287, 268, 114, 238, 317, 333, 155, 329, 273, 261, 9, 339, 87, 36, 285, 376, 171, 157, 144, 275, 52, 70, 337, 148, 78, 66, 371, 350, 27, 93, 160, 198, 55, 31, 97, 295, 265, 312, 311, 67, 120, 33, 225, 8, 19, 200, 356, 185, 104, 272, 1, 368, 16, 372, 122, 232, 257, 61, 105] + 12: [255, 35, 167, 22, 103, 306, 170, 75, 332, 152, 368, 10, 39, 101, 271, 129, 261, 318, 358, 357, 200, 231, 119, 256, 249, 325, 38, 121, 20, 293, 329, 148, 352, 44, 34, 343, 299, 247, 182, 269, 228, 88, 166, 66, 222, 220, 302, 42, 204, 177, 203, 381, 168, 296, 233, 369, 176, 195, 205, 348, 229, 277, 153, 102, 107, 135, 326, 14, 286, 142, 1, 315, 61, 62, 161, 321, 76, 154, 65, 32, 244, 217, 85, 252, 316, 113, 117, 320, 210, 145, 242, 100, 359, 239, 53, 225, 120, 82, 333, 48, 175, 289, 192, 361, 354, 339, 68, 194, 25, 60, 2, 300, 322, 294, 291, 157, 155, 376, 9, 191, 45, 138, 280, 58, 11, 40, 13, 183, 365, 263, 274, 310, 345, 131, 237, 125, 79, 26, 163, 206, 377, 240, 162, 272, 130, 223, 172, 363, 344, 349, 97, 181, 208, 295, 382, 330, 87, 16, 209, 371, 362, 63, 337, 23, 15, 201, 350, 41, 29, 298, 139, 187, 122, 159, 257, 86, 186, 128, 383, 105, 375, 55, 212, 258, 307, 338, 360, 213, 21, 331, 73, 110, 158, 127, 150, 49, 17, 124, 290, 51, 340, 134, 137, 215, 147, 89, 288, 116, 57, 164, 380, 74, 283, 253, 188, 114, 165, 67, 4, 173, 214, 270, 379, 112, 284, 207, 106, 146, 282, 96, 81, 230, 143, 94, 202, 248, 95, 56, 199, 372, 327, 303, 243, 156, 28, 109, 126, 50, 98, 259, 281, 216, 317, 108, 133, 160, 171, 180, 241, 59, 144, 193, 227, 52, 234, 224, 136, 232, 335, 115, 351, 12, 336, 246, 374, 70, 251, 178, 347, 236, 196, 111, 132, 328, 92, 324, 373, 250, 149, 190, 378, 46, 245, 267, 71, 90, 198, 123, 278, 346, 268, 323, 27, 33, 264, 197, 308, 31, 91, 304, 370, 185, 0, 84, 313, 37, 169, 235, 36, 140, 5, 297, 24, 174, 30, 8, 54, 72, 367, 184, 221, 342, 275, 64, 353, 366, 78, 226, 69, 341, 219, 292, 287, 151, 260, 3, 6, 276, 218, 266, 364, 265, 104, 273, 80, 238, 301, 309, 77, 254, 83, 189, 262, 356, 319, 355, 305, 334, 179, 312, 279, 285, 47, 93, 18, 211, 19, 7, 118, 314, 311, 141, 99, 43] + 13: [170, 274, 3, 41, 276, 330, 366, 370, 159, 176, 184, 31, 202, 143, 320, 6, 207, 355, 141, 145, 74, 333, 243, 221, 105, 161, 326, 84, 296, 50, 250, 72, 115, 150, 103, 138, 346, 101, 28, 278, 293, 236, 47, 351, 314, 126, 192, 175, 108, 23, 30, 77, 86, 135, 100, 310, 11, 306, 118, 58, 75, 273, 309, 67, 178, 300, 79, 179, 382, 361, 151, 177, 164, 303, 267, 379, 220, 305, 325, 308, 270, 217, 287, 104, 233, 383, 117, 63, 90, 181, 269, 354, 120, 277, 140, 155, 232, 328, 238, 57, 96, 231, 254, 197, 88, 171, 279, 295, 369, 356, 246, 292, 373, 378, 219, 237, 144, 283, 374, 129, 52, 8, 331, 153, 174, 257, 348, 71, 154, 234, 213, 214, 185, 167, 284, 212, 152, 224, 272, 368, 160, 66, 33, 230, 18, 149, 12, 194, 265, 282, 55, 102, 338, 162, 323, 227, 364, 275, 261, 158, 208, 22, 10, 43, 239, 225, 93, 183, 380, 34, 163, 106, 357, 268, 59, 113, 358, 13, 124, 119, 182, 372, 336, 352, 258, 347, 341, 156, 82, 193, 315, 316, 195, 14, 260, 81, 0, 201, 110, 17, 199, 46, 131, 45, 16, 289, 294, 281, 334, 65, 147, 62, 25, 262, 166, 327, 222, 125, 89, 68, 345, 92, 245, 187, 123, 19, 313, 190, 205, 85, 226, 60, 264, 20, 317, 148, 255, 247, 350, 78, 87, 114, 280, 9, 169, 21, 342, 271, 142, 298, 35, 198, 112, 297, 91, 229, 299, 210, 375, 321, 26, 291, 253, 256, 76, 241, 288, 248, 136, 259, 244, 133, 360, 73, 235, 353, 285, 157, 252, 203, 99, 251, 332, 200, 322, 165, 324, 7, 80, 329, 189, 318, 24, 172, 209, 134, 186, 109, 116, 111, 188, 69, 40, 363, 2, 42, 168, 240, 228, 196, 359, 139, 15, 223, 36, 337, 56, 242, 263, 311, 70, 38, 5, 365, 83, 4, 343, 191, 128, 97, 349, 302, 367, 206, 27, 32, 44, 362, 312, 137, 39, 51, 95, 340, 335, 304, 307, 64, 49, 376, 54, 266, 122, 98, 211, 339, 344, 107, 121, 146, 29, 37, 48, 286, 319, 127, 381, 53, 94, 377, 130, 61, 249, 180, 1, 215, 173, 371, 132, 290, 218, 301, 216, 204] + 14: [229, 355, 274, 8, 268, 348, 228, 95, 342, 154, 279, 41, 13, 158, 123, 319, 68, 37, 339, 286, 296, 90, 156, 237, 200, 189, 12, 183, 96, 285, 302, 329, 127, 74, 305, 179, 45, 366, 139, 140, 144, 65, 280, 53, 292, 249, 260, 174, 160, 209, 378, 261, 60, 239, 146, 141, 117, 371, 269, 44, 166, 33, 149, 163, 324, 145, 266, 169, 256, 226, 116, 221, 306, 316, 89, 282, 244, 376, 94, 321, 309, 267, 361, 190, 258, 7, 62, 180, 107, 219, 188, 303, 210, 351, 295, 289, 345, 2, 327, 147, 137, 17, 118, 142, 32, 97, 162, 250, 216, 197, 59, 246, 69, 370, 22, 333, 349, 227, 373, 105, 330, 225, 335, 148, 224, 251, 106, 214, 257, 365, 334, 19, 299, 80, 293, 185, 233, 252, 113, 54, 340, 67, 315, 165, 255, 30, 126, 56, 287, 383, 150, 99, 52, 39, 213, 111, 294, 208, 135, 206, 184, 159, 301, 133, 325, 234, 5, 377, 298, 0, 115, 191, 350, 240, 28, 6, 153, 195, 238, 16, 230, 132, 55, 367, 236, 283, 170, 131, 352, 98, 130, 331, 70, 110, 34, 71, 372, 49, 353, 23, 85, 177, 61, 182, 157, 270, 25, 328, 288, 326, 167, 104, 203, 245, 101, 347, 176, 253, 202, 242, 356, 248, 311, 173, 278, 338, 20, 9, 312, 31, 47, 35, 181, 307, 51, 243, 232, 119, 73, 50, 211, 204, 259, 308, 77, 112, 186, 254, 198, 231, 360, 21, 354, 220, 379, 343, 357, 193, 108, 346, 102, 109, 222, 129, 276, 359, 223, 152, 364, 43, 42, 262, 358, 36, 78, 58, 263, 281, 57, 92, 11, 247, 72, 27, 29, 138, 318, 88, 143, 76, 122, 275, 207, 175, 337, 235, 124, 64, 317, 271, 26, 125, 161, 215, 332, 369, 363, 382, 128, 178, 82, 380, 284, 86, 168, 264, 1, 63, 310, 273, 375, 46, 136, 14, 322, 194, 66, 84, 81, 320, 192, 368, 38, 196, 83, 121, 4, 15, 75, 155, 164, 199, 344, 151, 100, 3, 79, 374, 265, 314, 297, 323, 291, 201, 277, 40, 187, 172, 48, 241, 341, 336, 300, 313, 217, 93, 10, 114, 103, 171, 304, 290, 87, 18, 362, 24, 120, 381, 91, 134, 218, 272, 212, 205] + 15: [130, 258, 68, 259, 39, 236, 215, 80, 151, 56, 277, 174, 143, 164, 162, 78, 182, 247, 212, 86, 150, 282, 81, 43, 382, 347, 53, 137, 45, 37, 337, 116, 218, 91, 73, 129, 126, 380, 361, 299, 304, 110, 21, 336, 245, 144, 260, 270, 11, 200, 298, 295, 300, 159, 146, 237, 376, 83, 342, 177, 111, 115, 36, 47, 5, 194, 102, 263, 34, 331, 254, 165, 30, 226, 308, 208, 55, 64, 95, 213, 54, 79, 161, 158, 140, 280, 171, 44, 178, 243, 122, 220, 348, 322, 368, 276, 0, 355, 104, 350, 371, 326, 284, 60, 221, 16, 113, 311, 138, 24, 225, 335, 323, 305, 2, 248, 229, 50, 233, 253, 141, 306, 354, 244, 204, 71, 172, 257, 190, 147, 197, 188, 149, 310, 292, 154, 179, 286, 302, 345, 19, 6, 325, 85, 15, 155, 148, 76, 199, 332, 252, 20, 360, 48, 193, 135, 255, 379, 285, 319, 372, 42, 275, 142, 114, 69, 8, 12, 294, 67, 72, 234, 224, 211, 334, 265, 278, 210, 303, 262, 201, 327, 136, 98, 183, 108, 288, 249, 315, 75, 366, 328, 88, 324, 175, 340, 291, 231, 119, 9, 329, 40, 344, 163, 320, 235, 106, 357, 49, 145, 279, 309, 77, 96, 206, 13, 339, 118, 381, 289, 272, 92, 375, 167, 3, 239, 369, 370, 312, 7, 124, 181, 41, 307, 156, 214, 46, 25, 93, 356, 359, 251, 217, 261, 367, 185, 333, 90, 317, 23, 296, 256, 242, 196, 352, 351, 170, 238, 105, 117, 87, 268, 205, 341, 377, 349, 321, 267, 314, 169, 283, 316, 240, 65, 378, 184, 14, 187, 58, 112, 66, 103, 346, 219, 297, 152, 264, 281, 160, 362, 128, 290, 202, 38, 74, 121, 232, 139, 353, 168, 241, 109, 31, 338, 133, 28, 101, 59, 269, 10, 22, 250, 271, 313, 180, 61, 365, 123, 383, 358, 63, 273, 274, 89, 374, 176, 293, 222, 195, 120, 17, 125, 132, 27, 131, 127, 227, 57, 94, 134, 4, 189, 266, 203, 51, 157, 35, 173, 1, 18, 230, 207, 100, 209, 246, 166, 82, 343, 62, 363, 301, 191, 107, 97, 99, 216, 26, 373, 84, 52, 364, 153, 32, 29, 318, 330, 228, 223, 70, 287, 186, 33, 198, 192] + 16: [134, 118, 330, 185, 49, 342, 302, 383, 187, 235, 123, 294, 306, 2, 248, 73, 58, 153, 219, 356, 42, 301, 96, 188, 252, 200, 102, 333, 343, 109, 207, 363, 81, 373, 258, 371, 241, 61, 243, 63, 40, 254, 326, 361, 10, 320, 156, 249, 15, 303, 67, 331, 86, 318, 78, 283, 189, 270, 212, 65, 365, 307, 99, 21, 56, 380, 227, 70, 379, 9, 4, 106, 29, 336, 214, 337, 93, 105, 201, 62, 14, 128, 113, 110, 256, 138, 251, 355, 74, 329, 327, 362, 53, 94, 366, 257, 286, 44, 68, 125, 338, 289, 250, 298, 287, 87, 347, 285, 310, 97, 37, 103, 281, 311, 8, 167, 36, 160, 370, 124, 228, 358, 211, 242, 83, 378, 240, 163, 88, 284, 131, 117, 279, 210, 260, 6, 349, 136, 192, 204, 133, 111, 368, 353, 120, 7, 269, 11, 154, 129, 5, 374, 255, 213, 146, 346, 27, 314, 115, 38, 119, 139, 282, 34, 195, 45, 19, 339, 104, 264, 158, 18, 95, 272, 55, 193, 172, 127, 324, 168, 170, 359, 230, 288, 79, 351, 173, 197, 222, 161, 126, 1, 275, 92, 64, 271, 300, 143, 259, 224, 265, 141, 367, 16, 89, 321, 237, 51, 175, 155, 31, 76, 220, 122, 174, 238, 225, 354, 208, 54, 178, 304, 190, 266, 186, 157, 47, 261, 116, 50, 332, 169, 382, 130, 341, 72, 278, 191, 247, 98, 100, 262, 165, 280, 277, 144, 348, 183, 171, 364, 246, 180, 108, 177, 328, 377, 292, 159, 231, 182, 232, 334, 276, 22, 317, 151, 340, 313, 253, 267, 244, 30, 381, 166, 107, 20, 216, 77, 149, 352, 164, 12, 350, 179, 296, 345, 309, 344, 140, 234, 43, 162, 114, 90, 205, 35, 24, 295, 236, 199, 85, 84, 59, 316, 82, 0, 41, 184, 229, 202, 293, 322, 198, 221, 25, 57, 206, 319, 48, 142, 194, 233, 145, 268, 308, 176, 150, 33, 28, 112, 299, 263, 325, 13, 215, 3, 369, 152, 147, 75, 312, 273, 203, 274, 209, 121, 46, 226, 181, 52, 71, 335, 80, 66, 39, 305, 357, 372, 137, 297, 290, 375, 132, 60, 223, 323, 91, 196, 360, 315, 218, 17, 26, 217, 245, 101, 32, 23, 135, 291, 376, 69, 239, 148] + 17: [131, 383, 374, 226, 87, 311, 185, 113, 351, 6, 52, 96, 22, 97, 180, 98, 272, 165, 171, 332, 19, 116, 41, 363, 382, 89, 271, 379, 93, 244, 345, 48, 55, 326, 233, 163, 315, 378, 30, 66, 291, 334, 268, 301, 54, 69, 72, 121, 119, 15, 28, 137, 366, 136, 343, 114, 328, 234, 167, 339, 218, 145, 243, 10, 298, 282, 245, 191, 248, 173, 260, 204, 196, 186, 95, 360, 195, 310, 16, 112, 278, 364, 150, 164, 316, 23, 141, 257, 11, 285, 17, 281, 169, 152, 159, 170, 284, 200, 362, 208, 126, 78, 279, 266, 189, 175, 172, 39, 194, 381, 228, 103, 33, 214, 142, 259, 20, 45, 42, 49, 198, 341, 21, 94, 300, 102, 183, 217, 369, 76, 270, 32, 355, 149, 176, 349, 129, 306, 35, 321, 14, 122, 202, 203, 236, 108, 305, 138, 346, 219, 359, 5, 246, 347, 370, 222, 104, 199, 177, 313, 143, 84, 327, 168, 264, 13, 36, 336, 118, 221, 309, 181, 27, 240, 120, 329, 287, 372, 25, 318, 53, 83, 88, 201, 4, 368, 348, 322, 333, 86, 127, 235, 79, 303, 330, 307, 190, 254, 193, 197, 323, 133, 75, 8, 157, 73, 148, 211, 367, 139, 146, 162, 376, 166, 331, 147, 216, 377, 101, 338, 44, 238, 361, 85, 64, 308, 255, 350, 65, 132, 182, 109, 223, 178, 247, 51, 156, 123, 237, 251, 337, 188, 294, 80, 58, 91, 117, 232, 130, 29, 297, 56, 207, 335, 206, 273, 92, 135, 263, 70, 18, 61, 59, 269, 81, 187, 3, 342, 179, 249, 258, 161, 267, 158, 124, 31, 344, 352, 40, 230, 184, 242, 289, 111, 209, 174, 283, 312, 375, 250, 26, 105, 319, 357, 225, 275, 1, 324, 276, 290, 134, 296, 231, 380, 371, 256, 74, 9, 340, 47, 60, 7, 154, 365, 37, 253, 265, 262, 299, 210, 128, 106, 192, 63, 57, 153, 227, 277, 224, 160, 71, 353, 252, 293, 320, 325, 261, 220, 100, 212, 140, 12, 155, 286, 2, 125, 62, 107, 317, 46, 24, 38, 373, 99, 292, 314, 82, 304, 43, 215, 295, 115, 205, 213, 110, 302, 90, 239, 151, 274, 67, 50, 144, 241, 280, 356, 229, 288, 77, 0, 68, 34, 358, 354] + 18: [271, 169, 136, 26, 31, 144, 285, 3, 165, 46, 366, 73, 359, 233, 109, 22, 201, 50, 8, 176, 150, 9, 335, 329, 243, 179, 348, 82, 220, 311, 209, 76, 95, 354, 238, 294, 249, 290, 62, 198, 226, 222, 344, 1, 32, 59, 78, 147, 30, 286, 215, 110, 236, 68, 237, 297, 99, 326, 188, 274, 346, 339, 177, 149, 131, 0, 278, 58, 157, 349, 279, 81, 63, 267, 153, 325, 256, 175, 332, 259, 197, 375, 364, 86, 44, 352, 230, 18, 199, 323, 186, 80, 227, 382, 309, 340, 251, 65, 185, 193, 211, 347, 380, 189, 321, 306, 282, 266, 67, 70, 89, 41, 342, 83, 87, 334, 367, 232, 250, 235, 308, 205, 276, 132, 363, 148, 324, 127, 383, 158, 66, 224, 381, 2, 107, 85, 29, 377, 277, 336, 196, 353, 97, 242, 208, 245, 106, 356, 210, 154, 119, 116, 355, 376, 284, 330, 33, 124, 269, 371, 202, 331, 79, 225, 137, 7, 328, 289, 15, 125, 244, 12, 13, 351, 112, 108, 60, 216, 4, 56, 229, 312, 172, 90, 184, 14, 117, 72, 134, 379, 120, 268, 47, 300, 138, 260, 53, 28, 36, 174, 146, 139, 151, 35, 190, 98, 181, 20, 141, 283, 262, 88, 162, 288, 313, 102, 140, 178, 93, 145, 187, 94, 214, 213, 234, 338, 51, 126, 317, 75, 258, 115, 273, 24, 160, 173, 194, 49, 327, 293, 10, 257, 228, 272, 45, 315, 304, 142, 167, 34, 357, 341, 105, 103, 19, 365, 23, 307, 223, 263, 241, 350, 361, 39, 195, 38, 96, 182, 292, 270, 166, 303, 143, 170, 219, 130, 369, 180, 372, 161, 280, 345, 212, 206, 221, 55, 374, 314, 43, 217, 71, 118, 16, 207, 368, 192, 25, 360, 291, 378, 77, 204, 318, 337, 191, 104, 343, 129, 135, 159, 302, 152, 111, 264, 362, 40, 155, 287, 254, 48, 358, 64, 27, 17, 123, 218, 42, 370, 281, 5, 305, 252, 295, 91, 52, 156, 69, 320, 57, 6, 333, 113, 114, 239, 301, 373, 253, 246, 101, 240, 164, 171, 248, 121, 261, 316, 296, 122, 231, 61, 203, 54, 319, 265, 255, 322, 92, 275, 168, 200, 247, 163, 133, 298, 84, 37, 299, 100, 11, 21, 310, 128, 74, 183] + 19: [24, 207, 243, 302, 188, 219, 184, 218, 227, 159, 35, 158, 54, 71, 278, 173, 68, 251, 204, 2, 277, 273, 346, 237, 131, 75, 280, 146, 326, 212, 22, 186, 340, 351, 103, 112, 267, 39, 224, 86, 56, 226, 55, 142, 378, 355, 369, 375, 187, 313, 307, 111, 52, 183, 300, 323, 38, 97, 275, 58, 25, 335, 1, 134, 6, 76, 368, 257, 347, 215, 203, 18, 333, 202, 160, 3, 191, 383, 311, 32, 232, 174, 63, 37, 150, 269, 214, 252, 163, 236, 282, 78, 145, 93, 50, 81, 190, 330, 240, 206, 241, 322, 114, 363, 164, 271, 153, 264, 171, 238, 30, 295, 127, 172, 16, 272, 67, 345, 374, 125, 284, 262, 293, 26, 309, 242, 298, 87, 258, 235, 94, 59, 231, 289, 223, 304, 266, 248, 360, 104, 175, 109, 98, 107, 366, 338, 64, 179, 91, 126, 197, 328, 349, 77, 354, 44, 292, 122, 13, 222, 359, 83, 195, 138, 40, 31, 95, 287, 108, 380, 372, 62, 140, 329, 82, 139, 370, 247, 149, 73, 348, 123, 178, 367, 0, 5, 151, 72, 196, 229, 133, 198, 7, 120, 101, 339, 364, 291, 8, 113, 20, 303, 301, 217, 147, 135, 36, 166, 165, 96, 353, 221, 260, 249, 143, 84, 19, 23, 356, 65, 225, 362, 137, 209, 14, 208, 286, 294, 357, 162, 185, 21, 155, 261, 17, 199, 250, 305, 245, 253, 90, 377, 312, 211, 61, 161, 192, 66, 154, 296, 230, 189, 365, 15, 233, 352, 182, 167, 29, 283, 144, 4, 244, 325, 177, 45, 148, 47, 281, 119, 130, 168, 132, 234, 314, 315, 106, 11, 201, 51, 205, 308, 43, 220, 274, 317, 254, 263, 288, 57, 42, 141, 74, 255, 318, 373, 193, 334, 152, 79, 46, 246, 116, 310, 337, 270, 156, 239, 157, 110, 306, 336, 213, 10, 331, 129, 376, 321, 176, 320, 181, 332, 324, 361, 118, 124, 290, 100, 299, 379, 69, 327, 276, 350, 268, 319, 256, 265, 180, 115, 48, 70, 279, 216, 27, 33, 89, 117, 9, 41, 53, 285, 128, 228, 121, 49, 170, 297, 259, 342, 194, 85, 80, 60, 92, 88, 200, 341, 343, 99, 105, 28, 371, 381, 210, 358, 136, 344, 102, 169, 382, 316, 12, 34] + 20: [161, 174, 123, 307, 204, 330, 10, 273, 382, 140, 292, 92, 71, 322, 240, 35, 375, 314, 58, 18, 22, 86, 169, 15, 33, 73, 202, 89, 351, 210, 151, 115, 124, 329, 47, 136, 48, 234, 147, 103, 31, 298, 242, 27, 51, 52, 54, 61, 116, 4, 272, 287, 291, 32, 162, 164, 290, 96, 259, 258, 172, 126, 315, 194, 348, 26, 30, 359, 42, 256, 165, 246, 20, 78, 304, 231, 191, 94, 154, 156, 250, 321, 333, 67, 211, 64, 261, 149, 129, 152, 1, 197, 76, 107, 137, 167, 334, 87, 335, 14, 376, 130, 62, 371, 324, 128, 187, 342, 347, 65, 74, 284, 218, 361, 368, 213, 372, 238, 340, 294, 338, 106, 365, 312, 212, 159, 217, 352, 283, 28, 356, 72, 225, 133, 293, 192, 186, 155, 38, 269, 168, 179, 181, 189, 98, 99, 145, 37, 173, 112, 353, 188, 148, 141, 311, 362, 216, 101, 358, 357, 177, 328, 41, 36, 150, 208, 34, 166, 370, 310, 252, 39, 3, 374, 254, 316, 59, 199, 40, 7, 271, 16, 63, 373, 220, 222, 203, 44, 288, 200, 206, 214, 153, 193, 268, 122, 110, 226, 363, 185, 209, 229, 381, 180, 286, 49, 121, 6, 364, 235, 175, 146, 367, 119, 114, 60, 248, 377, 236, 257, 326, 337, 308, 56, 349, 21, 296, 163, 355, 19, 157, 383, 251, 275, 237, 69, 17, 249, 262, 263, 278, 66, 45, 8, 81, 2, 105, 138, 325, 111, 125, 264, 227, 132, 77, 90, 176, 343, 118, 0, 233, 300, 274, 346, 297, 29, 127, 170, 201, 24, 253, 285, 142, 289, 113, 85, 323, 305, 134, 68, 245, 301, 244, 109, 117, 260, 266, 281, 302, 144, 369, 282, 55, 339, 224, 270, 158, 91, 221, 50, 160, 184, 182, 239, 57, 267, 219, 25, 247, 178, 131, 215, 345, 12, 100, 332, 327, 84, 75, 80, 366, 120, 319, 205, 341, 83, 207, 108, 295, 306, 255, 143, 198, 277, 299, 303, 336, 82, 43, 46, 317, 379, 195, 318, 93, 228, 95, 9, 135, 320, 104, 190, 13, 378, 265, 53, 309, 279, 196, 243, 97, 5, 232, 380, 360, 223, 230, 139, 11, 79, 88, 102, 354, 276, 313, 183, 70, 23, 344, 171, 241, 280, 331, 350] + 21: [360, 80, 302, 206, 210, 364, 88, 345, 0, 240, 16, 26, 366, 69, 9, 189, 234, 325, 197, 336, 362, 318, 62, 286, 165, 30, 65, 301, 14, 102, 258, 335, 333, 305, 105, 342, 207, 130, 85, 37, 176, 6, 326, 95, 235, 311, 10, 17, 152, 158, 140, 247, 82, 208, 94, 377, 182, 320, 22, 349, 299, 280, 43, 23, 245, 131, 84, 18, 259, 262, 97, 303, 125, 378, 40, 100, 136, 104, 346, 48, 244, 334, 163, 241, 143, 41, 76, 56, 249, 106, 7, 116, 248, 327, 35, 58, 355, 4, 113, 256, 53, 185, 199, 232, 252, 250, 172, 351, 219, 227, 144, 220, 368, 229, 168, 1, 64, 223, 260, 159, 370, 5, 77, 193, 160, 198, 146, 276, 273, 381, 181, 253, 156, 74, 129, 180, 255, 31, 161, 166, 322, 332, 93, 123, 47, 254, 71, 236, 38, 218, 348, 214, 118, 149, 307, 147, 15, 194, 230, 111, 183, 243, 63, 49, 90, 309, 312, 28, 317, 323, 204, 171, 187, 170, 340, 300, 379, 110, 195, 164, 70, 330, 217, 314, 296, 142, 278, 188, 343, 11, 42, 264, 173, 308, 257, 190, 50, 112, 117, 277, 178, 238, 380, 19, 196, 225, 135, 221, 33, 174, 73, 267, 138, 60, 98, 192, 354, 133, 237, 148, 137, 203, 329, 216, 81, 83, 319, 294, 75, 13, 68, 114, 328, 145, 298, 212, 101, 127, 44, 306, 281, 371, 119, 46, 86, 87, 169, 226, 284, 51, 191, 128, 8, 25, 313, 141, 288, 270, 338, 331, 39, 304, 66, 383, 290, 154, 275, 266, 292, 269, 57, 261, 213, 134, 12, 315, 239, 374, 99, 324, 202, 242, 32, 222, 107, 167, 108, 321, 132, 20, 376, 153, 29, 92, 361, 120, 150, 279, 184, 109, 121, 293, 209, 367, 373, 89, 372, 358, 287, 246, 21, 356, 155, 3, 295, 341, 337, 24, 274, 126, 157, 27, 67, 382, 268, 2, 357, 54, 162, 265, 59, 231, 224, 283, 179, 339, 103, 316, 124, 289, 205, 200, 363, 45, 91, 177, 151, 365, 175, 344, 52, 72, 78, 233, 263, 352, 359, 139, 251, 291, 211, 347, 353, 115, 34, 79, 186, 36, 215, 271, 96, 310, 201, 122, 55, 375, 228, 285, 61, 282, 272, 297, 369, 350] + 22: [254, 376, 198, 34, 242, 112, 44, 355, 252, 70, 188, 111, 8, 222, 125, 170, 361, 366, 220, 233, 133, 367, 305, 200, 54, 256, 81, 300, 96, 353, 265, 246, 137, 282, 62, 257, 302, 150, 370, 304, 114, 270, 231, 10, 205, 2, 28, 79, 141, 224, 94, 126, 207, 184, 263, 244, 130, 348, 345, 311, 67, 83, 60, 185, 14, 3, 48, 25, 71, 12, 320, 318, 322, 283, 373, 241, 97, 136, 26, 103, 119, 182, 30, 91, 342, 344, 223, 6, 21, 146, 293, 234, 135, 249, 95, 101, 181, 24, 178, 204, 332, 73, 37, 329, 61, 55, 272, 218, 100, 378, 174, 193, 122, 228, 53, 258, 15, 99, 72, 47, 65, 189, 238, 45, 13, 271, 46, 208, 274, 151, 334, 129, 350, 128, 212, 88, 43, 158, 107, 243, 142, 280, 143, 154, 221, 148, 20, 33, 330, 115, 230, 235, 362, 317, 229, 149, 306, 161, 123, 214, 27, 113, 156, 49, 374, 210, 380, 87, 301, 199, 327, 299, 138, 314, 379, 262, 35, 160, 92, 145, 16, 168, 354, 162, 197, 163, 121, 368, 192, 287, 157, 217, 260, 209, 110, 9, 132, 164, 177, 139, 383, 240, 351, 338, 336, 68, 106, 116, 349, 358, 339, 191, 309, 165, 105, 248, 303, 269, 78, 32, 268, 202, 364, 279, 85, 187, 120, 152, 64, 69, 255, 23, 140, 277, 166, 56, 297, 289, 247, 250, 261, 278, 183, 371, 276, 38, 144, 29, 117, 352, 357, 7, 80, 372, 312, 369, 226, 175, 109, 18, 321, 59, 1, 39, 98, 227, 335, 382, 211, 215, 356, 298, 102, 219, 196, 179, 291, 90, 296, 326, 284, 295, 167, 86, 313, 292, 253, 267, 328, 343, 288, 316, 147, 42, 4, 169, 58, 333, 381, 285, 375, 195, 264, 17, 57, 75, 340, 63, 324, 225, 319, 341, 236, 124, 377, 180, 290, 77, 273, 176, 108, 266, 294, 190, 19, 104, 308, 5, 127, 66, 172, 194, 315, 323, 307, 310, 281, 216, 337, 51, 331, 201, 74, 118, 93, 50, 153, 251, 173, 76, 365, 259, 206, 347, 82, 40, 131, 41, 239, 159, 155, 22, 11, 0, 286, 89, 275, 237, 232, 186, 52, 36, 134, 171, 31, 363, 213, 84, 245, 203, 325, 360, 346, 359] + 23: [26, 135, 376, 19, 240, 324, 47, 374, 127, 148, 92, 340, 80, 70, 233, 23, 228, 292, 210, 298, 329, 231, 333, 223, 284, 258, 344, 164, 16, 14, 297, 239, 369, 214, 161, 9, 62, 165, 171, 299, 345, 89, 145, 186, 234, 189, 37, 64, 314, 167, 177, 211, 331, 203, 287, 94, 11, 218, 315, 263, 35, 213, 220, 149, 10, 290, 129, 157, 119, 66, 5, 307, 316, 76, 73, 200, 174, 68, 155, 236, 115, 104, 248, 61, 357, 31, 377, 343, 30, 117, 142, 380, 3, 216, 168, 288, 124, 187, 72, 274, 170, 24, 197, 154, 192, 182, 86, 41, 283, 326, 249, 191, 313, 224, 40, 222, 150, 96, 125, 178, 347, 352, 58, 363, 302, 206, 121, 349, 338, 139, 267, 46, 202, 34, 383, 181, 364, 42, 278, 198, 60, 366, 43, 81, 172, 132, 244, 281, 201, 269, 358, 176, 180, 59, 2, 56, 133, 146, 378, 91, 317, 237, 311, 356, 153, 160, 65, 301, 330, 250, 143, 109, 335, 212, 354, 362, 25, 375, 106, 188, 319, 88, 266, 116, 138, 208, 83, 217, 296, 4, 15, 242, 275, 100, 323, 120, 147, 82, 327, 261, 67, 372, 351, 130, 190, 144, 102, 312, 152, 272, 136, 361, 370, 17, 52, 325, 341, 204, 348, 280, 107, 7, 134, 99, 285, 355, 79, 367, 353, 230, 322, 247, 20, 141, 183, 293, 13, 251, 8, 235, 71, 175, 300, 225, 295, 87, 103, 156, 84, 55, 6, 371, 209, 50, 74, 36, 195, 342, 221, 44, 32, 365, 53, 137, 162, 193, 45, 0, 245, 108, 114, 243, 336, 373, 205, 49, 253, 276, 337, 334, 22, 318, 246, 112, 166, 29, 291, 265, 305, 264, 252, 69, 77, 173, 12, 232, 304, 57, 159, 78, 309, 111, 184, 229, 95, 194, 255, 101, 1, 93, 98, 207, 382, 196, 54, 128, 256, 21, 163, 268, 97, 33, 308, 140, 310, 277, 350, 259, 118, 289, 105, 262, 18, 110, 39, 286, 126, 328, 85, 51, 48, 306, 279, 27, 241, 169, 270, 257, 131, 379, 219, 151, 271, 38, 215, 359, 185, 254, 273, 179, 321, 303, 90, 339, 28, 346, 122, 294, 123, 227, 260, 282, 158, 368, 238, 360, 381, 63, 199, 332, 226, 113, 75, 320] + 24: [154, 357, 290, 172, 360, 112, 334, 186, 283, 159, 354, 91, 226, 274, 356, 49, 380, 3, 206, 63, 169, 295, 116, 11, 239, 60, 346, 282, 271, 296, 76, 161, 281, 160, 171, 291, 25, 18, 187, 240, 165, 316, 208, 364, 36, 374, 74, 81, 265, 263, 87, 259, 28, 286, 255, 230, 53, 135, 277, 382, 273, 260, 377, 39, 345, 118, 221, 321, 235, 323, 313, 280, 65, 225, 37, 170, 348, 10, 363, 320, 57, 331, 217, 317, 101, 68, 247, 250, 167, 340, 214, 17, 152, 133, 115, 166, 252, 372, 97, 210, 381, 64, 301, 207, 26, 204, 192, 15, 41, 106, 138, 336, 224, 307, 58, 297, 84, 168, 242, 315, 194, 30, 103, 4, 38, 14, 173, 249, 373, 163, 262, 342, 180, 184, 378, 300, 199, 347, 261, 40, 219, 158, 198, 241, 9, 22, 85, 328, 139, 361, 104, 375, 343, 78, 195, 306, 318, 254, 350, 96, 182, 12, 287, 83, 257, 233, 222, 56, 302, 134, 218, 203, 79, 88, 86, 188, 8, 148, 209, 44, 109, 236, 43, 174, 141, 123, 349, 82, 358, 179, 264, 298, 246, 6, 189, 111, 337, 324, 294, 146, 237, 24, 164, 190, 71, 355, 113, 149, 145, 245, 279, 376, 153, 269, 47, 51, 66, 181, 258, 267, 176, 120, 107, 284, 42, 67, 329, 238, 132, 256, 143, 228, 90, 227, 371, 100, 2, 21, 232, 314, 20, 92, 128, 110, 185, 27, 266, 62, 231, 213, 144, 61, 325, 308, 46, 351, 293, 77, 48, 59, 147, 278, 197, 1, 292, 215, 335, 333, 303, 150, 175, 94, 352, 359, 50, 234, 362, 367, 98, 162, 124, 285, 33, 305, 45, 127, 251, 353, 7, 52, 369, 95, 157, 105, 220, 370, 137, 202, 35, 54, 299, 131, 142, 177, 338, 205, 272, 140, 178, 13, 304, 244, 339, 270, 365, 72, 276, 196, 212, 126, 99, 191, 102, 193, 183, 80, 201, 332, 200, 243, 69, 322, 89, 129, 327, 379, 223, 0, 70, 211, 122, 121, 5, 29, 319, 31, 344, 366, 93, 156, 130, 73, 341, 310, 114, 326, 216, 16, 311, 268, 312, 108, 309, 275, 23, 253, 32, 151, 136, 19, 55, 288, 248, 125, 229, 75, 119, 289, 117, 330, 34, 155, 368, 383] + 25: [315, 30, 309, 77, 145, 305, 32, 120, 287, 215, 84, 240, 89, 343, 135, 359, 369, 304, 81, 130, 184, 202, 133, 14, 105, 300, 361, 127, 248, 22, 34, 122, 96, 293, 118, 349, 1, 286, 88, 104, 148, 106, 338, 233, 263, 123, 91, 158, 269, 168, 101, 2, 246, 232, 273, 102, 230, 342, 294, 197, 231, 153, 60, 97, 4, 291, 336, 237, 312, 259, 37, 28, 181, 355, 212, 251, 193, 260, 51, 281, 362, 283, 208, 289, 322, 265, 119, 172, 217, 272, 241, 194, 43, 19, 277, 326, 379, 192, 268, 375, 25, 376, 333, 7, 296, 378, 100, 131, 373, 76, 244, 146, 161, 143, 225, 319, 199, 345, 185, 328, 201, 94, 11, 356, 74, 166, 186, 177, 157, 21, 45, 40, 128, 137, 95, 57, 26, 301, 50, 329, 382, 239, 267, 178, 282, 372, 62, 298, 78, 0, 112, 5, 8, 318, 351, 381, 252, 377, 229, 54, 126, 218, 196, 347, 132, 221, 354, 290, 297, 257, 69, 370, 53, 27, 179, 271, 35, 325, 70, 210, 46, 236, 142, 63, 264, 71, 108, 198, 13, 270, 156, 220, 72, 23, 284, 258, 85, 39, 113, 49, 160, 38, 59, 262, 346, 9, 56, 256, 367, 321, 348, 242, 20, 203, 68, 55, 357, 331, 58, 310, 31, 200, 17, 358, 109, 163, 86, 124, 226, 175, 245, 155, 3, 147, 12, 285, 182, 6, 44, 121, 139, 276, 107, 98, 299, 61, 327, 10, 280, 66, 216, 18, 154, 222, 341, 80, 366, 279, 261, 140, 211, 295, 350, 204, 308, 134, 254, 144, 24, 79, 223, 243, 337, 75, 36, 167, 364, 371, 64, 41, 205, 360, 42, 90, 266, 170, 255, 165, 228, 180, 29, 83, 334, 162, 152, 171, 190, 33, 314, 368, 302, 234, 209, 99, 138, 227, 307, 114, 191, 278, 316, 324, 330, 365, 339, 149, 67, 238, 249, 275, 111, 292, 250, 183, 87, 311, 174, 224, 48, 141, 335, 169, 288, 151, 195, 129, 247, 317, 214, 65, 82, 332, 52, 73, 235, 313, 380, 207, 136, 306, 253, 125, 323, 164, 176, 103, 115, 352, 187, 383, 15, 213, 116, 274, 150, 159, 340, 110, 353, 117, 93, 320, 173, 16, 92, 188, 344, 363, 374, 219, 47, 206, 303, 189] + 26: [136, 150, 63, 162, 16, 183, 266, 223, 88, 167, 83, 247, 48, 138, 323, 353, 97, 103, 268, 365, 17, 113, 45, 0, 256, 75, 232, 301, 7, 49, 325, 225, 8, 64, 100, 29, 13, 216, 284, 330, 235, 310, 322, 132, 1, 54, 68, 86, 237, 53, 230, 214, 304, 289, 33, 137, 369, 249, 146, 328, 359, 148, 319, 190, 229, 174, 24, 221, 57, 318, 10, 73, 76, 32, 114, 182, 25, 219, 220, 340, 317, 43, 346, 244, 154, 91, 296, 4, 243, 222, 58, 155, 288, 96, 106, 111, 89, 18, 339, 233, 338, 116, 279, 38, 355, 158, 245, 82, 200, 311, 259, 321, 133, 102, 251, 294, 71, 104, 161, 35, 107, 350, 123, 143, 134, 264, 300, 352, 141, 109, 59, 77, 326, 101, 324, 14, 250, 345, 95, 248, 309, 139, 169, 170, 269, 124, 253, 271, 342, 120, 337, 130, 293, 315, 108, 382, 180, 31, 360, 262, 306, 99, 186, 343, 66, 215, 65, 98, 267, 50, 144, 372, 298, 336, 3, 140, 185, 335, 74, 112, 163, 119, 358, 145, 282, 28, 257, 210, 168, 175, 209, 218, 217, 11, 291, 320, 263, 254, 69, 308, 61, 135, 30, 62, 67, 176, 272, 151, 142, 152, 173, 194, 127, 239, 159, 202, 115, 231, 177, 312, 117, 368, 12, 105, 47, 40, 9, 303, 193, 199, 373, 208, 283, 46, 381, 273, 203, 228, 234, 242, 172, 380, 236, 281, 70, 224, 344, 316, 307, 376, 92, 131, 265, 198, 26, 191, 367, 27, 241, 118, 285, 277, 195, 212, 171, 37, 129, 383, 280, 157, 78, 362, 255, 72, 34, 331, 206, 205, 56, 313, 85, 147, 23, 379, 213, 246, 261, 305, 90, 292, 181, 5, 184, 149, 94, 341, 238, 226, 36, 110, 156, 204, 179, 287, 2, 327, 79, 51, 192, 15, 81, 290, 252, 196, 187, 363, 19, 302, 197, 364, 354, 334, 275, 361, 240, 274, 258, 377, 211, 164, 366, 207, 52, 260, 348, 351, 80, 6, 21, 349, 371, 160, 276, 278, 227, 22, 126, 332, 55, 128, 178, 166, 44, 356, 122, 201, 20, 357, 87, 378, 41, 370, 93, 153, 189, 329, 121, 125, 165, 347, 297, 39, 333, 314, 270, 42, 295, 188, 84, 286, 60, 299, 374, 375] + 27: [103, 354, 58, 164, 320, 181, 213, 143, 75, 303, 381, 339, 77, 89, 322, 118, 305, 7, 91, 54, 174, 218, 45, 276, 171, 199, 250, 365, 275, 105, 130, 259, 185, 65, 157, 289, 62, 283, 173, 138, 314, 57, 206, 200, 39, 53, 60, 137, 225, 73, 341, 301, 219, 204, 135, 358, 348, 237, 335, 290, 266, 99, 102, 23, 278, 317, 27, 88, 162, 329, 304, 296, 46, 362, 2, 9, 101, 95, 107, 159, 132, 96, 346, 74, 229, 364, 86, 334, 194, 148, 11, 92, 209, 203, 146, 163, 286, 168, 227, 81, 59, 71, 49, 24, 131, 222, 230, 48, 260, 196, 115, 83, 140, 141, 337, 155, 125, 10, 114, 215, 267, 18, 347, 28, 249, 367, 205, 35, 33, 184, 208, 264, 302, 372, 94, 166, 108, 160, 19, 169, 243, 212, 176, 220, 12, 265, 349, 34, 279, 269, 26, 128, 224, 112, 55, 178, 324, 216, 238, 78, 284, 356, 187, 161, 316, 379, 214, 274, 287, 43, 239, 318, 121, 167, 361, 79, 14, 280, 359, 50, 352, 16, 158, 345, 226, 170, 255, 363, 378, 235, 240, 244, 312, 61, 252, 117, 351, 300, 315, 366, 369, 190, 182, 21, 76, 149, 72, 32, 325, 191, 42, 254, 368, 326, 172, 5, 375, 106, 248, 66, 307, 0, 3, 256, 144, 231, 282, 223, 41, 330, 195, 192, 165, 311, 210, 298, 17, 63, 236, 262, 90, 263, 30, 104, 70, 374, 119, 376, 179, 336, 183, 323, 242, 383, 343, 82, 201, 193, 133, 370, 328, 271, 136, 268, 122, 67, 147, 44, 321, 217, 123, 288, 139, 281, 319, 8, 186, 142, 257, 64, 87, 331, 295, 371, 36, 93, 247, 310, 234, 4, 69, 145, 31, 350, 357, 202, 377, 97, 228, 85, 380, 232, 156, 308, 297, 109, 241, 261, 309, 6, 175, 327, 124, 129, 299, 120, 47, 353, 153, 116, 306, 113, 313, 293, 150, 110, 68, 189, 1, 56, 221, 355, 51, 100, 270, 338, 37, 177, 80, 211, 333, 233, 197, 292, 188, 207, 251, 272, 340, 258, 294, 332, 25, 373, 98, 15, 126, 246, 291, 127, 360, 111, 20, 38, 29, 13, 84, 151, 344, 152, 154, 22, 253, 382, 273, 198, 52, 245, 277, 40, 134, 180, 285, 342] + 28: [161, 0, 324, 120, 122, 7, 108, 191, 71, 58, 352, 339, 41, 240, 217, 367, 380, 43, 184, 78, 46, 245, 44, 98, 177, 36, 104, 363, 24, 294, 96, 80, 38, 284, 347, 232, 174, 140, 65, 53, 87, 262, 10, 18, 60, 86, 115, 150, 250, 25, 355, 77, 176, 288, 37, 75, 166, 223, 158, 63, 160, 28, 289, 105, 56, 19, 114, 341, 68, 282, 260, 264, 57, 159, 205, 296, 230, 133, 280, 331, 117, 213, 39, 131, 146, 29, 281, 277, 189, 254, 124, 62, 138, 156, 180, 207, 26, 302, 99, 312, 22, 226, 103, 307, 369, 378, 325, 204, 100, 157, 67, 337, 209, 271, 273, 127, 268, 134, 14, 183, 171, 208, 295, 375, 74, 66, 170, 73, 136, 89, 348, 225, 246, 229, 55, 76, 111, 126, 354, 164, 186, 147, 215, 257, 185, 306, 52, 4, 196, 152, 233, 224, 263, 219, 368, 51, 261, 90, 167, 163, 221, 182, 238, 278, 16, 1, 121, 365, 265, 64, 305, 148, 106, 210, 211, 151, 382, 364, 323, 110, 292, 276, 252, 101, 123, 141, 360, 198, 212, 173, 279, 311, 168, 255, 30, 197, 310, 203, 172, 113, 272, 50, 137, 227, 165, 235, 135, 220, 32, 340, 376, 48, 316, 91, 118, 142, 49, 81, 314, 143, 322, 12, 334, 154, 377, 218, 266, 251, 303, 237, 47, 373, 102, 304, 83, 309, 301, 190, 329, 336, 344, 317, 107, 321, 239, 188, 195, 253, 357, 128, 3, 228, 200, 88, 285, 326, 269, 13, 33, 193, 206, 178, 27, 244, 6, 249, 274, 362, 61, 258, 332, 297, 372, 330, 287, 293, 8, 17, 192, 35, 291, 371, 145, 313, 2, 40, 298, 338, 92, 70, 243, 241, 248, 328, 97, 93, 199, 236, 320, 42, 5, 129, 361, 242, 299, 216, 349, 315, 300, 333, 194, 256, 327, 358, 69, 381, 169, 270, 342, 34, 109, 31, 119, 345, 231, 72, 139, 267, 45, 202, 181, 318, 9, 54, 359, 343, 356, 153, 162, 286, 201, 283, 155, 214, 374, 346, 95, 59, 116, 383, 130, 370, 125, 20, 308, 247, 335, 222, 290, 112, 179, 23, 82, 21, 175, 15, 84, 366, 144, 79, 149, 351, 234, 85, 259, 275, 94, 350, 187, 319, 11, 132, 379, 353] + 29: [118, 304, 23, 283, 25, 129, 320, 346, 243, 87, 63, 204, 123, 50, 172, 218, 106, 329, 18, 272, 101, 353, 105, 217, 316, 242, 230, 28, 191, 56, 42, 166, 253, 270, 219, 308, 252, 30, 164, 189, 2, 130, 64, 138, 98, 206, 26, 363, 244, 295, 173, 271, 54, 339, 309, 160, 140, 134, 343, 145, 319, 278, 360, 85, 139, 222, 225, 59, 97, 3, 205, 317, 238, 31, 157, 135, 209, 368, 7, 250, 233, 62, 376, 71, 115, 246, 234, 201, 8, 5, 1, 268, 256, 367, 322, 29, 361, 203, 282, 383, 365, 333, 186, 266, 180, 375, 24, 68, 210, 350, 324, 215, 257, 349, 347, 228, 150, 292, 294, 328, 122, 224, 52, 46, 303, 220, 70, 338, 239, 194, 149, 17, 184, 296, 325, 254, 21, 27, 372, 342, 274, 37, 185, 240, 197, 198, 236, 255, 195, 379, 74, 16, 65, 48, 190, 216, 382, 38, 4, 378, 128, 273, 300, 146, 181, 380, 125, 340, 22, 229, 40, 10, 213, 327, 84, 161, 187, 121, 113, 99, 337, 297, 49, 41, 100, 226, 81, 221, 305, 6, 212, 285, 73, 323, 44, 89, 124, 76, 262, 148, 326, 57, 332, 163, 82, 86, 94, 259, 116, 232, 72, 171, 345, 104, 20, 371, 83, 169, 126, 369, 79, 103, 153, 261, 192, 179, 162, 334, 35, 251, 301, 279, 95, 245, 107, 321, 117, 144, 39, 91, 196, 277, 60, 11, 58, 120, 110, 351, 112, 366, 175, 237, 102, 357, 61, 298, 182, 178, 19, 280, 78, 310, 32, 167, 47, 36, 170, 302, 53, 287, 299, 13, 374, 227, 168, 0, 263, 92, 109, 223, 330, 286, 165, 335, 247, 214, 355, 152, 336, 137, 362, 127, 88, 354, 33, 202, 119, 174, 188, 275, 341, 293, 132, 248, 12, 200, 131, 90, 264, 142, 269, 344, 373, 312, 315, 147, 111, 114, 158, 199, 15, 69, 356, 154, 211, 141, 96, 358, 80, 75, 291, 281, 289, 249, 108, 66, 318, 51, 67, 311, 14, 258, 265, 208, 183, 159, 235, 370, 207, 193, 364, 314, 34, 156, 136, 276, 151, 267, 313, 77, 359, 348, 93, 377, 55, 381, 9, 288, 352, 307, 306, 43, 45, 290, 155, 133, 331, 260, 231, 241, 284, 143, 177, 176] + 30: [208, 76, 366, 225, 11, 144, 283, 250, 97, 341, 253, 256, 141, 136, 292, 98, 157, 278, 179, 335, 51, 310, 55, 48, 44, 374, 186, 121, 220, 139, 69, 85, 209, 314, 178, 352, 207, 265, 63, 61, 191, 238, 8, 21, 22, 31, 36, 40, 239, 333, 243, 59, 2, 45, 41, 315, 216, 240, 100, 58, 215, 226, 281, 298, 174, 91, 72, 114, 163, 286, 349, 181, 282, 360, 342, 377, 371, 161, 203, 327, 368, 122, 169, 330, 102, 338, 171, 93, 251, 301, 47, 88, 94, 101, 118, 123, 140, 35, 289, 213, 13, 378, 206, 28, 68, 303, 340, 74, 152, 275, 210, 204, 344, 369, 343, 261, 54, 383, 197, 129, 241, 73, 50, 145, 194, 183, 347, 326, 381, 5, 370, 355, 16, 113, 126, 313, 367, 182, 252, 305, 128, 134, 146, 185, 266, 52, 62, 184, 137, 78, 295, 223, 147, 302, 43, 221, 77, 192, 309, 382, 170, 160, 23, 353, 231, 142, 359, 111, 375, 248, 297, 25, 166, 202, 263, 32, 57, 125, 130, 291, 173, 120, 198, 0, 149, 228, 4, 70, 307, 189, 218, 219, 65, 214, 351, 133, 222, 242, 257, 188, 246, 17, 362, 159, 89, 1, 193, 236, 277, 260, 67, 60, 132, 329, 150, 376, 66, 293, 7, 92, 288, 337, 107, 267, 264, 83, 155, 82, 127, 233, 255, 300, 80, 328, 229, 27, 131, 331, 237, 259, 245, 53, 373, 254, 86, 75, 168, 321, 346, 212, 271, 49, 138, 336, 339, 299, 217, 18, 285, 200, 96, 167, 162, 14, 172, 180, 244, 284, 79, 110, 230, 247, 279, 99, 287, 273, 153, 19, 190, 106, 56, 372, 34, 87, 176, 354, 268, 269, 37, 358, 156, 124, 272, 135, 24, 234, 64, 33, 205, 312, 12, 357, 6, 9, 195, 274, 258, 365, 108, 84, 306, 319, 280, 109, 332, 290, 175, 154, 276, 324, 304, 116, 323, 117, 3, 15, 308, 270, 199, 39, 148, 30, 201, 363, 322, 334, 262, 187, 143, 158, 29, 196, 46, 361, 10, 164, 316, 224, 317, 177, 71, 151, 320, 296, 356, 325, 104, 38, 103, 20, 294, 211, 115, 249, 318, 311, 95, 350, 90, 348, 165, 364, 81, 105, 232, 112, 235, 42, 26, 119, 227, 379, 345, 380] + 31: [0, 185, 32, 36, 335, 95, 330, 209, 58, 253, 62, 102, 223, 82, 74, 17, 66, 122, 46, 15, 97, 383, 141, 49, 354, 237, 376, 266, 204, 224, 107, 210, 367, 315, 99, 113, 127, 197, 372, 27, 174, 257, 151, 165, 148, 229, 132, 207, 159, 239, 227, 241, 116, 25, 219, 39, 119, 171, 72, 355, 258, 145, 236, 63, 23, 243, 277, 31, 318, 225, 373, 16, 84, 361, 362, 6, 272, 228, 48, 50, 240, 129, 298, 321, 135, 7, 235, 370, 313, 359, 311, 104, 117, 71, 289, 329, 128, 115, 87, 265, 279, 326, 270, 328, 22, 333, 283, 338, 112, 296, 211, 136, 40, 366, 65, 322, 256, 374, 217, 92, 142, 153, 12, 305, 275, 351, 198, 103, 85, 309, 24, 234, 327, 156, 79, 126, 114, 357, 377, 230, 203, 180, 294, 350, 233, 143, 261, 331, 3, 133, 187, 317, 349, 60, 34, 299, 348, 290, 380, 353, 232, 245, 175, 205, 222, 202, 213, 21, 302, 73, 196, 183, 271, 346, 352, 304, 238, 375, 160, 157, 168, 381, 53, 193, 255, 184, 30, 161, 5, 154, 212, 347, 26, 252, 70, 125, 365, 364, 215, 150, 325, 155, 378, 164, 276, 189, 78, 111, 231, 55, 134, 182, 77, 287, 307, 118, 336, 195, 260, 273, 14, 177, 371, 281, 263, 42, 139, 340, 300, 64, 293, 382, 295, 18, 179, 86, 358, 343, 37, 149, 201, 158, 363, 320, 192, 379, 147, 267, 146, 186, 61, 360, 56, 269, 28, 249, 93, 38, 368, 57, 105, 316, 312, 288, 303, 291, 4, 51, 20, 323, 1, 130, 80, 308, 345, 9, 262, 109, 280, 274, 282, 167, 301, 163, 297, 200, 90, 246, 68, 324, 144, 248, 221, 292, 69, 339, 173, 140, 188, 75, 334, 214, 106, 254, 152, 54, 2, 88, 137, 41, 284, 123, 45, 8, 110, 108, 59, 43, 91, 194, 47, 162, 101, 178, 337, 356, 52, 44, 344, 242, 319, 94, 199, 67, 341, 120, 100, 166, 278, 76, 19, 121, 250, 268, 96, 206, 170, 208, 131, 286, 33, 244, 138, 98, 81, 169, 176, 190, 314, 310, 369, 29, 285, 11, 247, 83, 220, 264, 306, 226, 259, 251, 342, 191, 216, 181, 124, 35, 332, 218, 10, 13, 89, 172] + 32: [63, 146, 310, 66, 141, 204, 96, 126, 258, 109, 289, 171, 227, 104, 69, 64, 48, 237, 127, 240, 81, 337, 24, 349, 363, 311, 42, 102, 103, 197, 73, 374, 196, 203, 377, 31, 23, 321, 217, 338, 194, 162, 68, 14, 123, 169, 177, 181, 367, 166, 246, 209, 271, 2, 342, 319, 202, 211, 308, 232, 120, 189, 298, 71, 26, 25, 157, 150, 286, 53, 371, 129, 6, 340, 154, 172, 221, 132, 152, 304, 153, 264, 0, 355, 118, 224, 46, 368, 47, 207, 11, 27, 147, 244, 245, 251, 352, 51, 344, 234, 362, 130, 121, 41, 186, 283, 279, 116, 273, 345, 266, 91, 111, 50, 314, 281, 59, 142, 39, 3, 302, 10, 37, 193, 34, 82, 299, 124, 238, 222, 330, 201, 170, 208, 133, 259, 242, 112, 291, 226, 326, 167, 262, 265, 94, 163, 164, 322, 49, 229, 370, 12, 40, 65, 75, 78, 156, 252, 58, 301, 212, 354, 199, 213, 178, 230, 336, 29, 176, 247, 315, 56, 139, 339, 9, 99, 243, 365, 223, 35, 160, 52, 188, 60, 76, 95, 98, 295, 235, 333, 173, 296, 218, 241, 107, 275, 108, 179, 8, 257, 16, 89, 263, 140, 233, 335, 306, 254, 325, 36, 231, 28, 250, 293, 382, 353, 70, 4, 381, 77, 236, 5, 136, 284, 125, 376, 20, 356, 67, 79, 255, 100, 180, 18, 17, 272, 15, 54, 300, 312, 148, 190, 38, 288, 346, 106, 280, 274, 214, 348, 83, 97, 87, 113, 294, 313, 72, 85, 292, 269, 45, 253, 183, 351, 115, 276, 22, 198, 317, 110, 210, 307, 195, 105, 13, 134, 305, 278, 343, 137, 260, 358, 192, 290, 93, 90, 316, 373, 33, 347, 327, 155, 268, 323, 200, 219, 359, 225, 174, 143, 131, 228, 7, 369, 55, 303, 101, 332, 324, 182, 256, 249, 80, 30, 309, 378, 372, 144, 282, 43, 165, 248, 117, 261, 44, 138, 220, 364, 216, 161, 74, 159, 267, 114, 379, 375, 92, 285, 122, 329, 135, 187, 21, 360, 61, 380, 184, 151, 331, 19, 357, 168, 297, 185, 175, 206, 239, 86, 32, 350, 84, 205, 366, 62, 158, 320, 361, 191, 287, 341, 128, 270, 119, 149, 334, 1, 277, 215, 88, 318, 328, 145, 57, 383] + 33: [38, 307, 240, 331, 323, 39, 125, 146, 206, 335, 54, 33, 155, 40, 288, 126, 303, 173, 174, 347, 375, 157, 200, 369, 213, 43, 3, 28, 166, 248, 327, 20, 346, 132, 76, 332, 247, 353, 214, 171, 359, 2, 46, 0, 265, 293, 333, 372, 91, 115, 267, 140, 280, 121, 183, 348, 84, 266, 255, 204, 30, 227, 308, 217, 18, 351, 355, 254, 79, 262, 197, 163, 152, 170, 106, 71, 66, 26, 110, 67, 135, 74, 286, 291, 310, 118, 242, 363, 7, 228, 284, 47, 239, 129, 69, 142, 339, 343, 154, 51, 259, 158, 107, 314, 108, 5, 190, 141, 25, 193, 131, 9, 305, 211, 149, 4, 198, 246, 55, 373, 279, 364, 137, 371, 281, 70, 285, 219, 10, 316, 130, 381, 120, 62, 315, 165, 274, 44, 344, 357, 282, 12, 34, 220, 36, 81, 299, 328, 63, 31, 96, 179, 201, 352, 127, 376, 218, 151, 192, 289, 61, 354, 187, 109, 45, 80, 156, 370, 114, 60, 49, 330, 113, 65, 278, 41, 252, 182, 181, 85, 78, 257, 178, 59, 199, 253, 313, 294, 203, 159, 189, 277, 319, 148, 283, 167, 320, 42, 297, 68, 322, 226, 169, 273, 1, 210, 356, 222, 235, 144, 205, 230, 383, 196, 382, 367, 186, 234, 368, 292, 24, 52, 271, 261, 318, 298, 300, 168, 188, 312, 244, 175, 378, 374, 269, 57, 302, 272, 264, 287, 260, 229, 301, 86, 145, 153, 379, 73, 241, 164, 37, 270, 160, 150, 208, 268, 14, 290, 93, 138, 123, 276, 19, 50, 180, 13, 58, 249, 350, 366, 101, 324, 136, 194, 111, 11, 225, 245, 365, 337, 92, 250, 99, 172, 77, 143, 83, 95, 177, 380, 258, 89, 75, 377, 231, 342, 147, 139, 361, 233, 17, 338, 117, 334, 8, 116, 329, 16, 216, 22, 162, 104, 88, 212, 105, 15, 215, 349, 53, 112, 221, 128, 23, 207, 27, 119, 98, 326, 97, 275, 309, 296, 100, 223, 236, 102, 362, 64, 251, 94, 176, 191, 325, 82, 358, 72, 48, 21, 35, 87, 237, 133, 311, 232, 202, 90, 317, 124, 256, 209, 360, 340, 122, 345, 336, 56, 103, 29, 295, 6, 238, 243, 184, 224, 134, 341, 195, 306, 321, 304, 185, 263, 161, 32] + 34: [108, 44, 151, 135, 145, 240, 178, 246, 83, 271, 78, 162, 208, 290, 81, 256, 136, 133, 33, 181, 73, 0, 247, 263, 113, 140, 62, 298, 105, 266, 314, 13, 101, 226, 331, 205, 17, 285, 90, 1, 355, 378, 320, 7, 375, 14, 76, 119, 179, 334, 372, 357, 239, 321, 291, 258, 59, 288, 41, 95, 134, 373, 12, 159, 368, 206, 195, 241, 48, 287, 367, 230, 215, 294, 10, 358, 142, 296, 374, 219, 36, 286, 28, 109, 242, 342, 212, 22, 43, 182, 38, 235, 177, 211, 122, 132, 16, 330, 61, 324, 311, 185, 148, 146, 115, 238, 92, 65, 225, 153, 184, 326, 327, 199, 251, 67, 270, 209, 102, 308, 103, 164, 283, 371, 261, 64, 68, 228, 111, 353, 49, 117, 278, 253, 9, 54, 120, 188, 252, 309, 198, 213, 160, 189, 249, 126, 196, 382, 18, 304, 31, 360, 315, 295, 191, 6, 125, 201, 107, 356, 123, 197, 143, 217, 30, 227, 333, 169, 232, 154, 63, 127, 139, 272, 85, 88, 233, 161, 34, 255, 165, 26, 39, 104, 128, 156, 338, 106, 121, 32, 224, 193, 306, 299, 190, 29, 332, 354, 203, 194, 347, 79, 335, 267, 244, 21, 144, 174, 84, 183, 172, 336, 328, 231, 364, 170, 361, 180, 322, 277, 149, 383, 98, 192, 269, 262, 93, 319, 52, 370, 339, 141, 281, 292, 349, 303, 129, 66, 279, 202, 15, 259, 248, 207, 82, 158, 362, 167, 379, 2, 8, 300, 4, 124, 58, 53, 313, 187, 37, 280, 346, 72, 214, 35, 5, 80, 19, 318, 221, 186, 147, 268, 282, 310, 100, 380, 57, 173, 55, 171, 351, 96, 216, 94, 350, 273, 316, 229, 340, 69, 3, 237, 74, 25, 243, 138, 220, 97, 222, 11, 87, 91, 110, 302, 345, 24, 60, 70, 245, 150, 359, 116, 301, 348, 114, 323, 23, 46, 264, 275, 289, 265, 274, 75, 131, 250, 155, 112, 71, 130, 325, 234, 369, 343, 341, 363, 366, 337, 218, 47, 312, 223, 293, 40, 99, 284, 254, 168, 77, 200, 329, 210, 175, 166, 137, 317, 236, 27, 204, 89, 50, 352, 157, 42, 56, 45, 51, 152, 305, 86, 118, 307, 176, 260, 297, 20, 257, 163, 344, 276, 376, 377, 381, 365] + 35: [319, 68, 322, 220, 144, 141, 138, 31, 147, 114, 335, 362, 159, 275, 357, 296, 158, 5, 294, 137, 152, 11, 170, 74, 348, 264, 330, 99, 380, 73, 43, 176, 105, 189, 149, 112, 23, 354, 238, 83, 225, 219, 226, 39, 120, 304, 228, 177, 45, 295, 72, 353, 237, 56, 257, 38, 154, 15, 265, 50, 266, 27, 102, 168, 18, 61, 328, 108, 143, 90, 323, 57, 227, 91, 382, 273, 179, 196, 315, 378, 317, 151, 253, 22, 26, 262, 97, 101, 270, 252, 280, 119, 229, 381, 293, 343, 1, 367, 276, 48, 260, 211, 368, 129, 3, 224, 46, 307, 14, 222, 373, 160, 244, 308, 282, 372, 42, 320, 16, 172, 109, 214, 369, 383, 376, 19, 17, 88, 198, 292, 259, 245, 94, 121, 77, 10, 69, 310, 188, 210, 173, 236, 291, 359, 345, 297, 86, 281, 62, 254, 67, 153, 231, 78, 41, 326, 361, 136, 312, 267, 356, 36, 171, 30, 125, 95, 32, 350, 366, 202, 339, 24, 290, 142, 66, 344, 274, 332, 161, 370, 181, 155, 288, 107, 351, 233, 123, 258, 212, 199, 85, 284, 363, 131, 53, 80, 246, 303, 318, 4, 221, 239, 364, 349, 331, 54, 157, 321, 128, 81, 93, 213, 209, 337, 96, 287, 8, 305, 347, 166, 174, 269, 217, 25, 6, 365, 156, 29, 115, 49, 165, 167, 271, 33, 371, 336, 76, 2, 118, 200, 75, 218, 203, 340, 44, 130, 286, 21, 116, 206, 375, 194, 191, 126, 37, 360, 140, 132, 192, 215, 334, 201, 256, 261, 111, 145, 82, 204, 184, 127, 175, 84, 35, 63, 248, 278, 249, 195, 255, 139, 148, 327, 208, 0, 341, 64, 283, 60, 169, 299, 133, 309, 182, 58, 55, 47, 289, 186, 311, 234, 241, 329, 98, 134, 272, 9, 298, 190, 313, 300, 302, 235, 20, 180, 117, 216, 185, 301, 279, 338, 103, 87, 135, 377, 306, 250, 263, 150, 178, 333, 242, 51, 71, 124, 146, 13, 28, 205, 342, 113, 240, 379, 314, 324, 346, 355, 352, 164, 162, 52, 163, 187, 325, 230, 268, 104, 243, 122, 12, 92, 277, 110, 106, 34, 7, 70, 285, 374, 79, 65, 207, 316, 232, 89, 358, 183, 197, 223, 193, 247, 251, 59, 100, 40] + 36: [232, 372, 197, 348, 299, 296, 380, 374, 150, 314, 31, 45, 246, 231, 293, 13, 68, 245, 155, 168, 216, 228, 46, 310, 0, 52, 335, 244, 119, 208, 51, 209, 204, 338, 251, 357, 118, 110, 235, 151, 285, 89, 269, 160, 114, 64, 37, 123, 22, 10, 223, 382, 359, 210, 241, 90, 291, 47, 85, 65, 201, 67, 166, 55, 132, 91, 50, 339, 354, 174, 100, 177, 255, 187, 81, 344, 358, 78, 220, 58, 239, 189, 236, 383, 60, 120, 226, 87, 7, 34, 27, 106, 221, 43, 270, 170, 184, 266, 12, 370, 62, 230, 326, 373, 73, 5, 54, 32, 175, 362, 319, 49, 349, 95, 164, 20, 171, 179, 237, 224, 305, 229, 104, 17, 157, 129, 243, 135, 281, 6, 158, 199, 337, 21, 115, 350, 278, 96, 124, 218, 234, 113, 279, 273, 371, 238, 225, 131, 74, 328, 98, 56, 97, 308, 125, 261, 240, 334, 288, 282, 167, 105, 298, 342, 142, 117, 271, 277, 323, 264, 76, 28, 148, 263, 283, 18, 44, 247, 315, 173, 316, 137, 61, 355, 154, 303, 227, 181, 306, 145, 290, 302, 368, 304, 193, 107, 176, 2, 320, 99, 301, 205, 92, 331, 333, 108, 219, 139, 361, 121, 172, 289, 153, 140, 343, 364, 134, 71, 212, 36, 188, 267, 365, 29, 329, 295, 381, 198, 35, 307, 254, 213, 26, 128, 327, 222, 367, 186, 292, 360, 272, 57, 178, 192, 84, 59, 258, 274, 352, 79, 287, 156, 33, 112, 194, 94, 122, 284, 322, 183, 15, 280, 66, 195, 102, 200, 257, 379, 88, 149, 318, 356, 363, 109, 259, 19, 249, 215, 312, 253, 161, 203, 136, 286, 182, 317, 127, 375, 53, 144, 311, 252, 250, 260, 242, 369, 207, 133, 332, 42, 321, 309, 111, 345, 63, 297, 300, 276, 346, 147, 11, 116, 211, 30, 39, 206, 165, 23, 159, 48, 24, 70, 169, 77, 130, 347, 324, 8, 378, 330, 353, 4, 214, 268, 16, 313, 340, 341, 325, 294, 185, 101, 196, 40, 275, 152, 233, 93, 162, 262, 9, 191, 138, 103, 69, 3, 72, 366, 41, 202, 217, 180, 38, 143, 351, 80, 163, 146, 83, 190, 265, 248, 256, 75, 14, 126, 86, 1, 82, 141, 377, 376, 25, 336] + 37: [218, 213, 199, 48, 193, 223, 344, 253, 22, 139, 181, 18, 145, 19, 122, 27, 149, 82, 143, 108, 285, 356, 83, 337, 347, 313, 38, 4, 116, 160, 17, 170, 175, 35, 81, 62, 130, 119, 300, 89, 90, 68, 84, 31, 132, 236, 239, 288, 40, 188, 123, 314, 46, 272, 292, 361, 171, 245, 165, 204, 163, 133, 359, 208, 76, 173, 144, 61, 39, 333, 329, 126, 206, 121, 244, 94, 284, 42, 154, 13, 91, 216, 169, 297, 191, 96, 231, 155, 11, 220, 275, 232, 99, 189, 352, 383, 268, 366, 0, 273, 311, 205, 190, 72, 77, 140, 360, 88, 328, 343, 257, 150, 127, 228, 176, 6, 230, 141, 278, 29, 100, 224, 80, 380, 97, 24, 325, 187, 299, 10, 60, 137, 310, 198, 106, 324, 177, 45, 219, 211, 346, 209, 56, 75, 294, 200, 340, 66, 309, 52, 372, 290, 382, 114, 185, 182, 147, 93, 330, 221, 370, 287, 270, 168, 217, 36, 264, 201, 87, 55, 37, 321, 195, 279, 105, 351, 138, 368, 303, 78, 295, 374, 248, 57, 202, 197, 369, 246, 358, 251, 375, 112, 148, 222, 162, 364, 242, 306, 341, 179, 174, 86, 129, 7, 65, 117, 131, 192, 315, 229, 21, 196, 365, 41, 30, 134, 376, 49, 247, 128, 255, 241, 274, 5, 172, 124, 152, 286, 291, 283, 142, 146, 107, 326, 54, 159, 381, 25, 238, 120, 293, 158, 113, 71, 348, 125, 289, 342, 234, 16, 302, 353, 14, 233, 153, 164, 157, 316, 51, 74, 203, 296, 92, 111, 34, 335, 331, 269, 312, 271, 280, 210, 249, 254, 215, 332, 378, 339, 166, 184, 237, 28, 261, 186, 2, 23, 276, 226, 109, 58, 349, 319, 243, 178, 362, 305, 95, 156, 355, 252, 318, 235, 32, 277, 322, 320, 104, 64, 323, 73, 161, 103, 3, 136, 69, 262, 373, 102, 317, 267, 298, 256, 266, 350, 115, 101, 308, 225, 70, 227, 20, 250, 367, 50, 377, 8, 301, 379, 79, 240, 265, 151, 363, 44, 371, 59, 47, 9, 327, 194, 118, 345, 336, 85, 258, 1, 43, 338, 304, 135, 15, 167, 334, 207, 98, 212, 33, 354, 26, 110, 53, 260, 183, 259, 63, 357, 214, 281, 180, 307, 12, 282, 67, 263] + 38: [301, 282, 50, 94, 322, 32, 171, 145, 212, 54, 97, 199, 125, 352, 271, 208, 193, 262, 57, 184, 179, 143, 228, 31, 28, 242, 353, 2, 173, 214, 56, 11, 47, 186, 351, 363, 164, 254, 259, 207, 116, 365, 330, 304, 161, 46, 60, 185, 6, 154, 187, 348, 332, 274, 177, 19, 17, 328, 253, 319, 137, 341, 107, 239, 29, 358, 87, 349, 156, 48, 246, 117, 306, 134, 174, 139, 310, 334, 172, 382, 25, 204, 283, 13, 102, 240, 320, 163, 175, 22, 377, 370, 312, 176, 237, 251, 266, 51, 114, 4, 178, 292, 226, 49, 130, 166, 135, 119, 234, 327, 90, 222, 67, 95, 150, 144, 255, 136, 34, 188, 289, 170, 26, 272, 235, 337, 360, 371, 104, 100, 295, 297, 183, 3, 357, 157, 314, 291, 218, 160, 162, 270, 339, 325, 203, 372, 383, 10, 369, 123, 285, 220, 63, 27, 317, 268, 267, 286, 225, 355, 227, 69, 224, 105, 58, 256, 7, 229, 265, 260, 76, 33, 258, 40, 200, 313, 273, 375, 44, 374, 247, 236, 12, 326, 350, 20, 167, 198, 249, 318, 340, 380, 158, 284, 126, 62, 287, 118, 263, 309, 230, 381, 279, 344, 108, 338, 106, 99, 14, 73, 362, 359, 88, 264, 368, 109, 276, 190, 84, 149, 103, 81, 296, 122, 74, 82, 215, 168, 35, 290, 91, 113, 24, 209, 281, 83, 127, 346, 38, 52, 378, 201, 189, 89, 213, 245, 354, 248, 379, 244, 0, 77, 257, 16, 288, 75, 324, 72, 59, 36, 68, 165, 96, 298, 5, 141, 197, 180, 15, 78, 302, 195, 43, 347, 61, 80, 55, 202, 70, 335, 18, 232, 315, 110, 155, 121, 64, 101, 221, 124, 366, 333, 93, 66, 86, 194, 39, 92, 373, 206, 53, 329, 361, 275, 147, 21, 321, 45, 211, 216, 142, 37, 342, 241, 356, 181, 131, 250, 23, 182, 261, 120, 311, 294, 71, 299, 192, 65, 364, 128, 153, 196, 1, 133, 98, 191, 269, 280, 30, 277, 223, 345, 8, 115, 293, 217, 278, 169, 300, 331, 233, 219, 146, 85, 9, 307, 323, 308, 367, 111, 151, 148, 376, 41, 305, 152, 238, 79, 129, 231, 252, 205, 138, 343, 316, 112, 42, 132, 303, 336, 140, 243, 159, 210] + 39: [282, 36, 176, 116, 153, 218, 292, 329, 204, 98, 15, 163, 146, 381, 352, 127, 31, 278, 331, 358, 276, 5, 226, 33, 72, 289, 193, 4, 243, 377, 169, 120, 58, 335, 189, 266, 206, 0, 197, 210, 326, 124, 12, 308, 29, 148, 361, 209, 310, 93, 100, 264, 1, 13, 155, 244, 297, 374, 141, 369, 51, 200, 84, 274, 340, 112, 190, 131, 62, 366, 63, 301, 212, 311, 241, 354, 338, 309, 114, 265, 283, 192, 287, 222, 78, 246, 252, 10, 219, 129, 294, 73, 154, 11, 52, 143, 247, 38, 85, 180, 348, 215, 279, 187, 145, 277, 172, 76, 178, 87, 375, 8, 248, 251, 138, 47, 152, 199, 364, 249, 14, 102, 45, 133, 128, 6, 130, 214, 166, 109, 288, 168, 157, 160, 175, 229, 233, 147, 205, 105, 332, 90, 321, 22, 66, 312, 313, 35, 223, 139, 242, 272, 270, 71, 295, 319, 77, 23, 202, 118, 48, 122, 225, 69, 367, 150, 322, 213, 182, 228, 95, 173, 230, 16, 220, 194, 376, 217, 196, 303, 108, 305, 123, 92, 255, 351, 32, 318, 57, 115, 165, 149, 104, 65, 314, 280, 234, 151, 119, 382, 174, 333, 250, 330, 49, 7, 235, 170, 191, 299, 101, 74, 50, 177, 307, 161, 134, 355, 88, 320, 275, 30, 363, 181, 60, 291, 19, 111, 337, 231, 211, 359, 238, 136, 370, 316, 344, 186, 334, 44, 372, 286, 132, 99, 110, 25, 360, 271, 296, 324, 40, 341, 365, 81, 298, 259, 42, 323, 239, 34, 126, 135, 371, 162, 302, 94, 106, 203, 164, 254, 306, 183, 91, 70, 27, 103, 46, 347, 37, 64, 342, 158, 67, 224, 184, 261, 9, 89, 236, 258, 379, 96, 82, 156, 325, 328, 80, 362, 17, 24, 208, 290, 227, 53, 232, 28, 336, 137, 353, 188, 144, 75, 345, 293, 185, 97, 357, 79, 68, 207, 171, 343, 41, 159, 55, 20, 18, 253, 339, 349, 125, 237, 269, 201, 21, 198, 56, 315, 43, 268, 368, 113, 267, 86, 2, 26, 121, 300, 3, 59, 273, 383, 140, 54, 263, 260, 378, 281, 317, 179, 373, 61, 167, 285, 221, 195, 240, 327, 216, 245, 346, 117, 107, 83, 142, 39, 284, 350, 256, 262, 356, 380, 304, 257] + 40: [31, 359, 367, 52, 153, 64, 74, 3, 91, 363, 336, 341, 322, 278, 200, 188, 137, 44, 99, 189, 25, 155, 339, 333, 177, 304, 301, 139, 241, 122, 141, 125, 135, 16, 174, 106, 119, 180, 39, 87, 255, 191, 72, 5, 92, 149, 248, 300, 319, 71, 33, 260, 204, 283, 160, 128, 164, 58, 97, 28, 303, 136, 321, 289, 27, 236, 205, 259, 307, 150, 138, 94, 103, 29, 356, 281, 273, 293, 222, 246, 78, 121, 361, 13, 32, 182, 34, 111, 325, 315, 109, 292, 220, 225, 144, 105, 15, 328, 346, 280, 55, 101, 247, 8, 332, 372, 59, 228, 275, 326, 271, 166, 211, 12, 221, 239, 56, 57, 178, 377, 192, 120, 206, 329, 37, 18, 233, 291, 118, 256, 84, 266, 117, 40, 218, 231, 114, 313, 223, 298, 83, 126, 69, 343, 68, 142, 179, 357, 70, 373, 265, 161, 369, 81, 90, 17, 1, 168, 274, 185, 11, 199, 382, 46, 187, 263, 244, 290, 294, 324, 93, 243, 171, 314, 261, 6, 147, 212, 184, 268, 49, 113, 242, 22, 173, 96, 176, 24, 172, 335, 353, 376, 175, 360, 108, 379, 286, 62, 297, 215, 88, 235, 351, 210, 284, 143, 63, 80, 54, 38, 148, 227, 53, 364, 43, 110, 2, 338, 305, 282, 19, 131, 345, 238, 270, 14, 276, 310, 50, 66, 77, 217, 234, 152, 381, 86, 272, 146, 312, 194, 378, 349, 21, 226, 169, 75, 26, 279, 158, 198, 100, 311, 296, 89, 209, 61, 193, 216, 196, 262, 316, 95, 82, 130, 7, 264, 318, 237, 48, 47, 366, 36, 375, 288, 348, 383, 140, 203, 257, 104, 352, 342, 45, 245, 320, 240, 331, 214, 317, 250, 358, 340, 213, 67, 337, 156, 254, 251, 224, 159, 362, 129, 162, 277, 145, 334, 302, 269, 79, 365, 299, 306, 165, 287, 167, 323, 350, 60, 73, 201, 115, 186, 0, 102, 368, 230, 35, 134, 354, 4, 380, 112, 20, 197, 170, 249, 207, 183, 190, 132, 181, 98, 107, 202, 232, 208, 258, 347, 9, 10, 219, 253, 30, 308, 370, 229, 309, 41, 65, 151, 355, 327, 371, 330, 195, 51, 85, 76, 116, 133, 23, 124, 374, 295, 163, 285, 154, 267, 157, 252, 127, 42, 344, 123] + 41: [49, 190, 119, 335, 27, 86, 365, 293, 79, 24, 206, 306, 348, 127, 140, 126, 217, 301, 379, 260, 156, 340, 133, 288, 66, 107, 6, 130, 315, 303, 359, 154, 100, 165, 13, 248, 113, 155, 67, 20, 183, 145, 189, 362, 382, 233, 247, 216, 378, 184, 159, 367, 103, 255, 273, 92, 111, 358, 302, 82, 383, 3, 219, 194, 349, 102, 125, 93, 229, 325, 330, 380, 101, 181, 208, 160, 10, 192, 15, 300, 179, 180, 207, 38, 147, 182, 238, 47, 201, 240, 211, 356, 304, 230, 296, 278, 199, 355, 197, 122, 11, 249, 50, 97, 187, 161, 311, 108, 45, 4, 202, 243, 308, 64, 225, 256, 235, 117, 242, 8, 63, 290, 277, 261, 220, 214, 37, 342, 39, 65, 72, 332, 148, 137, 29, 274, 267, 319, 270, 16, 58, 118, 227, 309, 170, 142, 369, 215, 166, 57, 84, 76, 172, 279, 12, 95, 318, 222, 264, 163, 336, 70, 54, 276, 258, 167, 112, 210, 363, 153, 18, 43, 173, 89, 41, 188, 236, 381, 234, 105, 310, 28, 265, 329, 331, 88, 177, 372, 291, 52, 263, 357, 60, 283, 275, 185, 71, 150, 200, 285, 136, 314, 262, 44, 326, 351, 25, 23, 134, 116, 282, 55, 237, 289, 224, 266, 176, 186, 19, 213, 244, 146, 59, 272, 352, 0, 141, 68, 73, 175, 171, 370, 110, 323, 377, 109, 203, 333, 31, 120, 350, 114, 366, 305, 376, 307, 168, 96, 34, 69, 347, 337, 104, 53, 286, 218, 252, 56, 313, 254, 271, 115, 144, 328, 151, 295, 81, 317, 61, 128, 123, 292, 1, 191, 259, 48, 257, 204, 375, 373, 121, 343, 106, 341, 281, 228, 22, 85, 374, 298, 129, 87, 196, 90, 223, 124, 7, 360, 334, 198, 9, 324, 42, 353, 26, 32, 226, 83, 294, 138, 250, 268, 299, 284, 91, 94, 74, 221, 99, 75, 78, 77, 297, 149, 316, 209, 14, 245, 62, 2, 162, 346, 169, 135, 40, 132, 321, 269, 30, 205, 251, 212, 368, 327, 364, 21, 280, 253, 80, 35, 239, 36, 174, 345, 241, 46, 322, 344, 339, 51, 131, 338, 246, 5, 371, 354, 232, 33, 143, 287, 320, 158, 98, 195, 231, 17, 139, 157, 312, 361, 193, 164, 178, 152] + 42: [123, 184, 212, 41, 349, 55, 281, 327, 351, 370, 29, 135, 28, 329, 36, 92, 33, 362, 380, 241, 301, 225, 323, 183, 226, 61, 251, 156, 364, 84, 284, 82, 360, 23, 288, 347, 96, 353, 107, 14, 271, 163, 273, 242, 25, 71, 95, 254, 152, 93, 154, 50, 106, 151, 150, 337, 377, 208, 21, 235, 248, 30, 67, 17, 173, 216, 314, 316, 257, 192, 113, 311, 62, 89, 315, 58, 205, 378, 278, 26, 194, 267, 90, 363, 124, 334, 22, 85, 190, 110, 78, 43, 91, 289, 294, 324, 117, 218, 2, 1, 303, 116, 264, 262, 263, 209, 189, 373, 328, 221, 169, 344, 147, 181, 322, 52, 223, 245, 101, 132, 307, 238, 49, 310, 196, 16, 270, 293, 120, 206, 98, 73, 185, 136, 207, 133, 276, 79, 54, 142, 240, 352, 369, 339, 31, 155, 333, 375, 114, 211, 63, 374, 220, 6, 342, 191, 275, 320, 140, 187, 87, 326, 335, 280, 118, 149, 32, 60, 8, 371, 260, 130, 34, 298, 355, 269, 233, 175, 272, 197, 104, 285, 304, 309, 153, 12, 11, 81, 317, 365, 46, 126, 359, 172, 256, 210, 148, 228, 350, 72, 195, 306, 255, 122, 379, 131, 4, 239, 230, 201, 356, 77, 366, 39, 160, 340, 137, 372, 9, 139, 232, 258, 291, 68, 112, 213, 229, 321, 128, 292, 330, 367, 253, 35, 70, 159, 382, 66, 361, 376, 217, 176, 158, 188, 42, 357, 215, 179, 243, 200, 48, 265, 358, 354, 259, 224, 214, 161, 103, 164, 45, 76, 381, 18, 146, 180, 64, 244, 299, 313, 332, 125, 300, 261, 290, 145, 368, 75, 186, 383, 69, 115, 83, 182, 0, 15, 24, 27, 94, 345, 111, 37, 20, 305, 279, 282, 162, 129, 59, 127, 277, 274, 178, 236, 287, 165, 308, 38, 202, 227, 65, 80, 237, 219, 246, 109, 99, 5, 336, 53, 346, 343, 199, 102, 312, 296, 144, 174, 88, 47, 57, 331, 13, 86, 40, 121, 266, 268, 325, 138, 295, 119, 247, 168, 222, 108, 203, 319, 348, 141, 341, 286, 167, 105, 338, 143, 234, 166, 297, 51, 19, 231, 283, 157, 97, 7, 204, 170, 171, 56, 318, 250, 10, 3, 198, 302, 134, 249, 100, 44, 177, 74, 252, 193] + 43: [168, 359, 298, 262, 278, 112, 20, 103, 152, 55, 127, 313, 307, 321, 4, 222, 282, 204, 306, 26, 36, 67, 334, 148, 293, 180, 56, 74, 9, 370, 295, 329, 144, 353, 118, 134, 211, 357, 341, 69, 7, 141, 383, 104, 25, 166, 331, 44, 225, 17, 30, 344, 308, 35, 13, 94, 131, 186, 322, 227, 198, 121, 89, 191, 238, 275, 339, 271, 160, 365, 155, 243, 60, 38, 45, 251, 309, 343, 159, 304, 267, 76, 19, 173, 265, 316, 162, 114, 41, 228, 42, 133, 350, 54, 189, 354, 375, 335, 297, 27, 332, 197, 349, 366, 57, 337, 111, 264, 1, 147, 368, 150, 90, 346, 277, 299, 149, 84, 196, 107, 49, 229, 143, 120, 263, 288, 170, 122, 8, 65, 151, 233, 290, 240, 311, 124, 106, 236, 208, 333, 33, 310, 192, 95, 66, 372, 75, 369, 6, 142, 59, 254, 22, 81, 138, 323, 274, 269, 374, 2, 232, 327, 83, 123, 296, 130, 241, 145, 99, 47, 210, 284, 101, 174, 116, 207, 294, 132, 158, 268, 300, 110, 260, 230, 62, 249, 270, 58, 289, 361, 206, 214, 98, 92, 283, 23, 177, 68, 216, 252, 326, 377, 29, 360, 305, 202, 255, 272, 154, 246, 234, 140, 5, 61, 10, 63, 185, 165, 217, 46, 128, 325, 175, 72, 279, 87, 287, 358, 355, 187, 285, 248, 172, 281, 340, 139, 244, 184, 373, 245, 315, 347, 190, 164, 82, 129, 364, 239, 363, 93, 223, 203, 338, 136, 379, 32, 378, 48, 286, 135, 367, 257, 237, 345, 24, 50, 253, 167, 220, 226, 18, 64, 71, 117, 193, 109, 320, 179, 312, 199, 328, 21, 362, 91, 176, 231, 195, 3, 221, 156, 14, 351, 181, 371, 37, 97, 235, 291, 342, 318, 108, 78, 250, 73, 205, 88, 219, 200, 261, 157, 194, 273, 102, 113, 266, 213, 125, 80, 31, 292, 356, 163, 381, 119, 330, 258, 51, 276, 0, 137, 352, 96, 212, 302, 169, 348, 105, 171, 324, 43, 161, 218, 39, 100, 314, 153, 301, 178, 224, 280, 85, 77, 52, 380, 86, 115, 34, 79, 28, 12, 242, 183, 16, 70, 146, 256, 215, 11, 259, 126, 247, 209, 188, 336, 201, 53, 317, 303, 40, 15, 319, 376, 182, 382] + 44: [213, 101, 238, 272, 60, 257, 252, 325, 93, 86, 118, 329, 144, 337, 276, 138, 350, 290, 150, 70, 130, 380, 87, 134, 162, 136, 83, 159, 243, 309, 211, 237, 230, 345, 49, 114, 245, 375, 373, 99, 247, 176, 293, 31, 58, 75, 209, 106, 77, 379, 284, 167, 110, 227, 29, 135, 30, 280, 289, 63, 102, 217, 355, 117, 244, 299, 173, 201, 132, 286, 202, 369, 107, 46, 370, 40, 59, 318, 341, 113, 234, 109, 367, 65, 35, 80, 239, 210, 283, 260, 297, 52, 76, 181, 339, 116, 301, 222, 223, 97, 178, 180, 43, 140, 11, 187, 182, 306, 304, 275, 158, 265, 85, 175, 163, 364, 79, 338, 54, 120, 36, 126, 15, 185, 149, 231, 23, 20, 72, 193, 4, 51, 255, 200, 195, 119, 298, 268, 356, 111, 88, 371, 13, 332, 191, 171, 170, 0, 351, 188, 277, 279, 143, 98, 212, 198, 228, 125, 343, 91, 224, 61, 154, 39, 71, 378, 174, 177, 327, 344, 196, 74, 205, 347, 32, 305, 203, 319, 204, 62, 273, 216, 5, 115, 28, 330, 37, 152, 148, 16, 161, 322, 199, 172, 346, 17, 300, 84, 68, 331, 166, 146, 242, 294, 365, 360, 14, 340, 192, 24, 122, 287, 285, 160, 269, 381, 128, 220, 278, 334, 105, 323, 81, 3, 263, 357, 256, 137, 312, 1, 27, 151, 363, 302, 47, 73, 316, 281, 295, 374, 10, 274, 168, 64, 155, 190, 82, 311, 236, 383, 333, 142, 33, 95, 324, 262, 6, 186, 219, 56, 352, 366, 359, 44, 157, 41, 92, 90, 214, 377, 169, 104, 241, 9, 53, 183, 2, 253, 22, 251, 372, 307, 123, 127, 353, 308, 112, 303, 317, 362, 21, 25, 271, 215, 328, 368, 34, 320, 249, 264, 358, 225, 89, 121, 266, 296, 55, 100, 131, 248, 67, 221, 292, 354, 108, 165, 270, 19, 336, 179, 235, 18, 314, 232, 42, 45, 153, 50, 240, 69, 139, 335, 133, 208, 189, 12, 226, 348, 145, 258, 382, 267, 376, 229, 288, 313, 141, 38, 48, 291, 94, 207, 184, 254, 349, 315, 66, 164, 96, 261, 342, 26, 147, 282, 8, 78, 156, 218, 57, 250, 321, 7, 124, 326, 233, 194, 246, 103, 259, 361, 197, 310, 129, 206] + 45: [92, 95, 219, 284, 120, 375, 286, 165, 128, 183, 228, 223, 16, 104, 250, 112, 80, 213, 157, 315, 236, 4, 340, 65, 34, 205, 38, 264, 231, 255, 204, 105, 15, 147, 93, 136, 332, 31, 382, 200, 265, 259, 75, 237, 299, 361, 100, 174, 140, 27, 164, 318, 317, 274, 369, 354, 61, 349, 234, 254, 307, 158, 277, 70, 313, 267, 86, 108, 175, 67, 22, 253, 58, 116, 46, 358, 29, 322, 127, 288, 142, 289, 368, 202, 225, 90, 62, 143, 270, 366, 311, 184, 11, 240, 138, 64, 178, 8, 329, 301, 144, 25, 176, 378, 47, 348, 52, 275, 239, 162, 155, 74, 12, 327, 249, 217, 292, 96, 1, 370, 330, 126, 87, 5, 150, 333, 381, 374, 363, 48, 78, 343, 238, 321, 89, 82, 353, 53, 216, 325, 49, 72, 355, 221, 10, 201, 276, 173, 134, 139, 171, 103, 269, 124, 224, 39, 118, 310, 229, 245, 84, 7, 345, 44, 32, 2, 123, 218, 344, 170, 304, 320, 294, 379, 342, 177, 79, 203, 371, 362, 337, 323, 56, 152, 159, 194, 6, 335, 195, 17, 117, 83, 227, 334, 33, 383, 316, 182, 356, 262, 21, 163, 380, 258, 187, 196, 36, 88, 54, 357, 135, 281, 279, 109, 261, 257, 35, 19, 252, 145, 37, 211, 263, 314, 0, 119, 3, 377, 111, 214, 153, 167, 193, 295, 77, 28, 60, 69, 244, 347, 154, 42, 367, 191, 188, 346, 141, 146, 298, 212, 272, 331, 359, 45, 326, 372, 328, 251, 242, 94, 106, 208, 303, 222, 20, 207, 319, 291, 283, 241, 151, 285, 220, 341, 113, 71, 73, 102, 280, 97, 232, 352, 287, 99, 130, 181, 137, 266, 23, 293, 373, 81, 110, 197, 339, 85, 180, 297, 107, 248, 168, 122, 51, 30, 308, 306, 360, 179, 260, 24, 302, 50, 76, 133, 290, 59, 9, 273, 57, 129, 166, 309, 268, 190, 91, 125, 18, 186, 206, 101, 278, 66, 235, 14, 40, 192, 121, 161, 350, 324, 233, 199, 247, 160, 131, 114, 209, 351, 198, 55, 115, 338, 68, 210, 41, 156, 26, 365, 376, 63, 43, 230, 256, 296, 98, 282, 169, 271, 246, 312, 364, 215, 243, 132, 189, 226, 305, 148, 13, 172, 149, 185, 336, 300] + 46: [142, 355, 106, 201, 46, 238, 6, 301, 60, 114, 199, 3, 85, 62, 117, 260, 132, 118, 192, 298, 23, 250, 237, 193, 383, 373, 78, 297, 189, 9, 265, 145, 283, 262, 244, 371, 43, 358, 330, 126, 196, 296, 185, 100, 178, 365, 122, 175, 183, 253, 367, 188, 240, 51, 324, 312, 40, 165, 2, 320, 282, 280, 33, 372, 334, 315, 109, 13, 41, 272, 267, 158, 377, 329, 92, 24, 222, 56, 317, 241, 91, 370, 257, 52, 138, 275, 341, 363, 269, 29, 116, 151, 225, 82, 379, 356, 352, 380, 354, 210, 327, 202, 332, 12, 187, 55, 44, 219, 215, 131, 195, 227, 232, 197, 137, 50, 362, 273, 103, 59, 111, 308, 97, 169, 223, 69, 293, 342, 177, 93, 98, 150, 163, 310, 31, 236, 1, 174, 276, 70, 14, 147, 5, 213, 311, 286, 149, 335, 233, 96, 221, 344, 271, 38, 18, 146, 102, 30, 143, 120, 80, 218, 156, 316, 63, 95, 205, 157, 45, 101, 263, 289, 119, 318, 159, 277, 261, 217, 309, 171, 208, 15, 77, 366, 381, 48, 181, 47, 349, 270, 220, 27, 133, 58, 230, 66, 368, 170, 246, 284, 136, 68, 303, 203, 350, 351, 108, 155, 99, 353, 295, 166, 176, 128, 172, 340, 336, 307, 328, 360, 313, 173, 348, 20, 333, 90, 361, 234, 337, 112, 127, 229, 300, 134, 251, 54, 0, 28, 231, 36, 10, 256, 207, 152, 19, 25, 274, 154, 124, 254, 345, 76, 79, 211, 8, 376, 287, 121, 180, 37, 129, 294, 198, 17, 125, 226, 35, 144, 89, 161, 7, 74, 359, 190, 285, 339, 347, 135, 258, 281, 87, 73, 306, 123, 84, 167, 252, 88, 247, 139, 110, 319, 314, 16, 239, 104, 194, 204, 113, 242, 266, 228, 130, 22, 326, 168, 259, 369, 304, 53, 292, 235, 268, 32, 141, 290, 299, 346, 71, 322, 264, 291, 86, 278, 26, 67, 75, 94, 288, 105, 212, 179, 200, 182, 325, 302, 34, 64, 343, 140, 224, 249, 191, 209, 214, 378, 338, 374, 39, 49, 115, 83, 186, 245, 42, 4, 164, 162, 216, 321, 57, 81, 206, 160, 255, 382, 243, 153, 364, 331, 184, 305, 375, 65, 21, 11, 72, 107, 279, 248, 357, 148, 323, 61] + 47: [195, 307, 347, 357, 193, 309, 215, 196, 21, 40, 342, 63, 313, 95, 68, 166, 231, 65, 184, 51, 101, 173, 252, 183, 139, 211, 163, 281, 37, 107, 132, 297, 150, 275, 57, 325, 108, 339, 277, 88, 270, 167, 125, 254, 169, 7, 369, 326, 3, 29, 137, 171, 119, 61, 314, 353, 274, 287, 188, 363, 251, 351, 69, 104, 249, 324, 284, 199, 0, 138, 233, 112, 315, 80, 15, 289, 276, 306, 115, 20, 97, 160, 48, 282, 383, 174, 58, 220, 202, 244, 99, 165, 13, 272, 191, 246, 343, 158, 134, 179, 35, 170, 90, 294, 362, 240, 59, 153, 114, 330, 84, 219, 151, 228, 370, 197, 296, 148, 39, 210, 234, 141, 243, 28, 225, 60, 341, 260, 130, 190, 152, 302, 245, 5, 133, 222, 47, 299, 283, 206, 182, 83, 156, 293, 54, 22, 75, 192, 279, 280, 345, 316, 198, 168, 346, 367, 98, 226, 93, 301, 223, 285, 323, 255, 205, 17, 4, 354, 378, 336, 348, 147, 259, 111, 185, 32, 81, 9, 242, 216, 358, 374, 16, 105, 76, 142, 116, 217, 221, 311, 214, 360, 44, 154, 300, 136, 371, 34, 14, 257, 36, 110, 118, 218, 159, 176, 26, 377, 124, 263, 298, 77, 203, 25, 45, 27, 53, 91, 321, 213, 273, 258, 200, 157, 144, 10, 365, 266, 74, 290, 30, 317, 145, 11, 268, 256, 241, 318, 236, 56, 352, 162, 382, 331, 291, 227, 333, 320, 175, 129, 131, 123, 212, 250, 269, 67, 224, 103, 140, 364, 322, 355, 52, 106, 23, 356, 71, 126, 328, 332, 329, 186, 253, 46, 120, 72, 177, 380, 86, 38, 372, 264, 161, 295, 8, 113, 239, 288, 24, 181, 359, 187, 43, 1, 286, 127, 344, 267, 327, 229, 122, 230, 155, 135, 42, 41, 33, 50, 12, 338, 180, 117, 366, 18, 121, 303, 292, 278, 49, 310, 334, 102, 19, 308, 207, 350, 100, 178, 85, 64, 78, 232, 261, 128, 62, 373, 312, 87, 271, 143, 375, 109, 305, 368, 79, 376, 238, 349, 235, 265, 208, 262, 89, 319, 172, 73, 204, 237, 340, 164, 96, 201, 70, 335, 337, 94, 31, 2, 248, 149, 189, 55, 379, 92, 304, 361, 247, 194, 82, 6, 66, 381, 146, 209] + 48: [17, 52, 85, 288, 65, 358, 99, 90, 83, 369, 235, 48, 325, 252, 290, 219, 161, 25, 75, 88, 118, 31, 172, 218, 22, 100, 158, 63, 374, 40, 206, 259, 382, 272, 214, 69, 37, 87, 309, 148, 154, 153, 124, 68, 103, 199, 209, 196, 271, 276, 78, 9, 117, 226, 121, 164, 268, 375, 223, 264, 185, 380, 368, 312, 322, 192, 247, 73, 89, 212, 275, 231, 307, 70, 49, 36, 269, 367, 346, 107, 82, 71, 366, 29, 165, 335, 66, 356, 86, 345, 251, 122, 337, 348, 245, 227, 140, 239, 141, 159, 270, 150, 236, 58, 33, 6, 5, 372, 342, 134, 232, 222, 130, 291, 233, 45, 255, 24, 168, 57, 317, 281, 72, 204, 280, 12, 55, 274, 178, 256, 95, 93, 143, 277, 350, 149, 171, 260, 365, 126, 265, 355, 304, 114, 328, 273, 381, 297, 77, 109, 19, 74, 157, 189, 177, 254, 34, 305, 378, 300, 7, 293, 112, 324, 162, 221, 230, 79, 202, 113, 289, 81, 84, 18, 98, 137, 243, 286, 194, 326, 323, 213, 228, 191, 8, 377, 332, 383, 364, 131, 1, 250, 132, 330, 353, 263, 249, 301, 234, 295, 133, 287, 139, 363, 186, 354, 371, 210, 246, 299, 174, 123, 341, 111, 4, 13, 379, 351, 119, 97, 43, 349, 144, 35, 39, 173, 310, 313, 27, 294, 238, 225, 298, 60, 183, 262, 197, 333, 30, 308, 267, 102, 101, 237, 26, 145, 343, 182, 28, 104, 327, 208, 316, 181, 155, 229, 179, 50, 120, 190, 170, 357, 315, 42, 339, 14, 138, 302, 352, 53, 169, 59, 216, 32, 135, 373, 224, 61, 201, 376, 331, 282, 248, 23, 220, 167, 136, 314, 283, 128, 311, 15, 20, 279, 44, 2, 21, 166, 240, 38, 142, 41, 320, 156, 193, 257, 370, 0, 160, 242, 347, 360, 180, 241, 96, 11, 198, 127, 292, 94, 261, 47, 217, 3, 56, 105, 110, 188, 338, 106, 296, 176, 303, 258, 10, 129, 76, 266, 284, 62, 51, 215, 116, 203, 207, 147, 344, 359, 115, 184, 319, 205, 175, 152, 285, 340, 329, 321, 318, 91, 64, 80, 146, 253, 211, 16, 151, 195, 108, 187, 92, 67, 163, 278, 244, 361, 362, 125, 46, 54, 306, 336, 200, 334] + 49: [302, 37, 209, 115, 212, 208, 148, 60, 91, 74, 381, 264, 94, 48, 97, 362, 80, 56, 277, 29, 4, 81, 139, 179, 35, 338, 172, 85, 71, 355, 84, 192, 143, 145, 168, 316, 7, 341, 287, 21, 249, 230, 32, 68, 135, 356, 123, 191, 321, 157, 197, 304, 181, 78, 261, 201, 177, 133, 260, 194, 30, 332, 195, 366, 336, 310, 16, 138, 365, 34, 41, 273, 371, 352, 98, 227, 114, 176, 358, 12, 31, 190, 154, 364, 88, 343, 87, 372, 334, 161, 141, 263, 129, 188, 300, 64, 297, 305, 242, 251, 294, 59, 353, 25, 276, 325, 107, 132, 187, 45, 314, 333, 144, 174, 137, 221, 140, 170, 189, 296, 301, 82, 327, 134, 351, 244, 53, 117, 200, 151, 228, 22, 52, 232, 166, 51, 165, 162, 171, 350, 203, 258, 50, 322, 253, 155, 9, 193, 83, 65, 293, 241, 90, 44, 292, 220, 344, 8, 100, 122, 328, 254, 340, 368, 240, 128, 89, 222, 225, 303, 238, 178, 103, 345, 280, 378, 46, 2, 342, 256, 125, 324, 27, 315, 363, 102, 236, 207, 291, 3, 347, 375, 126, 75, 271, 210, 15, 234, 116, 110, 180, 262, 380, 289, 312, 255, 95, 5, 205, 73, 63, 245, 196, 198, 160, 163, 146, 10, 308, 62, 359, 239, 72, 119, 0, 204, 101, 283, 182, 326, 57, 93, 367, 266, 99, 152, 252, 26, 370, 38, 374, 268, 313, 233, 346, 136, 213, 40, 335, 39, 218, 246, 120, 331, 131, 109, 121, 1, 17, 23, 223, 214, 224, 248, 376, 173, 33, 43, 307, 339, 47, 226, 106, 49, 285, 108, 299, 104, 357, 167, 237, 159, 379, 281, 247, 288, 373, 317, 79, 6, 202, 184, 319, 150, 66, 278, 275, 199, 76, 243, 329, 175, 183, 235, 250, 169, 382, 86, 311, 290, 112, 127, 55, 216, 67, 330, 156, 142, 320, 130, 61, 124, 282, 361, 348, 185, 111, 147, 42, 96, 231, 215, 383, 219, 24, 92, 360, 272, 19, 149, 257, 158, 105, 270, 153, 259, 286, 306, 269, 186, 298, 164, 58, 279, 28, 54, 70, 318, 323, 77, 20, 295, 267, 36, 309, 14, 211, 69, 118, 206, 284, 113, 354, 274, 377, 13, 229, 18, 265, 217, 11, 369, 349, 337] + 50: [86, 69, 26, 33, 140, 370, 125, 304, 329, 256, 295, 184, 15, 133, 122, 71, 165, 375, 219, 189, 180, 310, 81, 64, 160, 105, 278, 347, 362, 153, 339, 349, 156, 141, 354, 162, 291, 209, 21, 201, 216, 31, 120, 52, 131, 23, 50, 345, 268, 215, 350, 57, 137, 367, 287, 280, 327, 334, 114, 130, 121, 274, 301, 239, 186, 194, 290, 53, 276, 267, 73, 152, 242, 340, 108, 5, 297, 4, 344, 7, 305, 212, 90, 36, 82, 40, 381, 103, 328, 158, 144, 115, 307, 222, 55, 365, 29, 188, 27, 128, 311, 229, 262, 373, 377, 296, 88, 79, 272, 270, 363, 196, 313, 19, 348, 359, 247, 208, 94, 92, 163, 255, 211, 223, 37, 39, 337, 259, 63, 308, 68, 150, 323, 147, 77, 10, 246, 245, 104, 161, 118, 336, 241, 93, 100, 210, 325, 129, 226, 333, 352, 146, 175, 220, 217, 314, 253, 213, 6, 24, 380, 75, 203, 134, 252, 250, 139, 303, 107, 126, 99, 113, 135, 127, 204, 177, 200, 320, 237, 355, 285, 11, 324, 13, 47, 360, 279, 309, 254, 364, 182, 111, 316, 315, 214, 3, 235, 38, 284, 366, 312, 199, 0, 22, 155, 249, 353, 96, 48, 228, 183, 102, 206, 197, 85, 234, 322, 145, 172, 251, 236, 191, 281, 123, 106, 46, 383, 148, 171, 87, 346, 61, 157, 70, 142, 167, 227, 368, 257, 136, 109, 32, 56, 1, 25, 67, 218, 273, 62, 338, 341, 119, 271, 159, 298, 275, 149, 117, 263, 240, 319, 342, 371, 74, 20, 91, 28, 185, 300, 202, 292, 170, 351, 238, 89, 258, 65, 112, 101, 76, 173, 72, 261, 221, 260, 59, 283, 230, 49, 376, 232, 18, 80, 97, 361, 35, 9, 45, 193, 331, 224, 335, 132, 41, 244, 58, 116, 168, 43, 174, 151, 326, 277, 78, 357, 143, 178, 369, 288, 42, 34, 372, 330, 318, 98, 179, 30, 83, 207, 265, 266, 225, 302, 66, 332, 374, 181, 166, 321, 231, 299, 14, 233, 110, 282, 192, 358, 51, 343, 16, 187, 164, 306, 195, 154, 379, 169, 269, 382, 378, 294, 2, 205, 356, 243, 17, 289, 124, 198, 12, 248, 293, 138, 286, 60, 176, 8, 190, 95, 54, 317, 44, 84, 264] + 51: [24, 12, 229, 73, 16, 182, 64, 65, 60, 235, 105, 253, 49, 247, 374, 23, 275, 74, 341, 366, 192, 34, 309, 195, 361, 320, 289, 332, 115, 89, 328, 278, 319, 339, 43, 240, 83, 354, 158, 277, 70, 133, 185, 244, 21, 226, 326, 348, 186, 208, 126, 189, 111, 52, 136, 62, 104, 301, 304, 168, 11, 5, 216, 47, 245, 142, 86, 329, 280, 91, 321, 44, 236, 265, 69, 383, 315, 381, 114, 242, 93, 109, 53, 25, 66, 9, 257, 305, 355, 227, 302, 347, 76, 255, 156, 202, 372, 323, 82, 106, 98, 284, 176, 211, 270, 124, 129, 177, 346, 92, 35, 85, 212, 308, 237, 107, 279, 351, 204, 251, 10, 362, 324, 311, 94, 258, 379, 191, 307, 273, 55, 167, 125, 148, 28, 272, 154, 132, 175, 274, 225, 267, 327, 249, 102, 200, 144, 286, 371, 113, 228, 72, 87, 32, 330, 146, 269, 314, 259, 20, 285, 344, 172, 268, 18, 181, 59, 160, 166, 41, 224, 171, 100, 312, 350, 338, 145, 363, 336, 378, 184, 1, 230, 165, 174, 217, 179, 266, 22, 38, 121, 139, 137, 110, 287, 31, 206, 193, 180, 264, 135, 203, 233, 153, 246, 263, 8, 294, 292, 382, 97, 162, 380, 310, 370, 119, 50, 48, 234, 317, 369, 365, 27, 54, 276, 333, 159, 112, 218, 57, 4, 14, 199, 376, 187, 352, 36, 231, 151, 290, 373, 210, 17, 108, 6, 99, 356, 300, 343, 95, 250, 283, 130, 26, 46, 220, 331, 161, 81, 198, 219, 190, 79, 252, 141, 3, 205, 262, 196, 349, 170, 223, 80, 340, 197, 232, 19, 116, 7, 215, 0, 120, 293, 29, 117, 377, 2, 138, 375, 37, 342, 322, 101, 357, 122, 67, 140, 260, 188, 295, 103, 335, 299, 118, 281, 303, 254, 56, 42, 209, 207, 33, 368, 164, 306, 84, 313, 297, 45, 194, 78, 298, 58, 238, 149, 214, 39, 296, 77, 128, 71, 63, 147, 131, 155, 261, 221, 241, 337, 163, 178, 123, 127, 358, 152, 40, 325, 282, 243, 134, 13, 143, 183, 271, 345, 359, 360, 222, 90, 248, 75, 239, 157, 364, 213, 334, 51, 30, 288, 353, 61, 316, 15, 68, 96, 88, 201, 367, 169, 318, 150, 256, 173, 291] + 52: [381, 209, 121, 372, 130, 289, 345, 82, 41, 33, 26, 269, 349, 50, 325, 379, 222, 105, 227, 23, 134, 146, 301, 331, 213, 333, 326, 268, 58, 110, 59, 152, 318, 249, 143, 342, 114, 4, 353, 276, 297, 203, 159, 142, 317, 199, 204, 176, 99, 239, 92, 217, 151, 323, 34, 298, 382, 60, 44, 168, 252, 98, 88, 133, 221, 191, 281, 160, 116, 280, 3, 154, 279, 208, 45, 225, 2, 277, 244, 21, 139, 347, 303, 197, 111, 96, 358, 363, 9, 185, 202, 144, 373, 264, 212, 180, 334, 336, 327, 216, 63, 214, 206, 77, 183, 68, 80, 192, 54, 354, 37, 171, 378, 5, 296, 30, 233, 292, 361, 319, 39, 287, 40, 93, 201, 177, 294, 189, 179, 100, 356, 322, 226, 311, 181, 112, 36, 187, 19, 15, 86, 272, 370, 247, 102, 137, 52, 332, 31, 73, 321, 193, 184, 190, 140, 196, 210, 236, 70, 312, 285, 286, 315, 124, 248, 369, 72, 17, 374, 337, 306, 308, 89, 242, 282, 84, 220, 129, 188, 118, 344, 313, 229, 302, 145, 255, 65, 122, 240, 360, 162, 348, 107, 103, 25, 231, 267, 362, 290, 211, 78, 346, 357, 219, 355, 95, 257, 123, 256, 234, 341, 223, 364, 300, 76, 53, 69, 66, 27, 161, 352, 383, 20, 238, 51, 136, 28, 310, 340, 343, 293, 147, 132, 314, 265, 338, 14, 295, 16, 305, 157, 283, 48, 307, 320, 246, 195, 263, 7, 172, 245, 259, 266, 8, 291, 108, 339, 166, 83, 173, 75, 243, 113, 377, 170, 324, 109, 119, 270, 200, 365, 207, 230, 330, 101, 380, 18, 126, 304, 275, 158, 42, 273, 241, 1, 376, 46, 175, 71, 97, 186, 61, 182, 85, 174, 163, 13, 62, 260, 56, 87, 149, 309, 228, 74, 117, 359, 167, 367, 6, 153, 351, 316, 55, 125, 235, 57, 258, 148, 194, 47, 198, 49, 141, 43, 299, 135, 232, 261, 79, 11, 366, 254, 94, 218, 375, 104, 165, 205, 120, 371, 250, 271, 169, 90, 288, 10, 128, 32, 64, 35, 155, 127, 262, 178, 215, 29, 67, 328, 115, 22, 91, 253, 350, 237, 284, 224, 251, 150, 368, 156, 278, 81, 335, 138, 274, 329, 131, 12, 0, 24, 164, 106, 38] + 53: [110, 363, 118, 160, 309, 237, 225, 134, 269, 127, 62, 12, 361, 77, 313, 197, 259, 159, 175, 146, 111, 339, 57, 370, 136, 358, 251, 101, 275, 7, 14, 43, 177, 1, 74, 34, 267, 212, 98, 286, 223, 307, 167, 185, 84, 114, 210, 272, 240, 311, 186, 68, 247, 280, 345, 92, 180, 268, 162, 130, 125, 24, 325, 94, 381, 383, 152, 300, 176, 89, 122, 362, 369, 216, 375, 189, 236, 333, 299, 59, 258, 21, 201, 289, 323, 48, 39, 64, 163, 260, 211, 165, 11, 191, 314, 312, 73, 354, 4, 8, 66, 173, 214, 155, 145, 360, 194, 103, 368, 284, 241, 168, 346, 341, 188, 324, 277, 47, 15, 143, 328, 120, 2, 166, 96, 215, 198, 294, 0, 45, 53, 80, 331, 83, 298, 106, 303, 263, 288, 213, 33, 99, 377, 316, 38, 109, 32, 217, 222, 142, 273, 227, 132, 6, 291, 26, 285, 326, 5, 200, 25, 338, 205, 378, 373, 281, 72, 282, 226, 128, 174, 65, 158, 278, 133, 220, 242, 257, 108, 380, 29, 199, 342, 113, 372, 356, 41, 27, 204, 245, 265, 18, 16, 296, 330, 75, 270, 164, 365, 42, 302, 202, 376, 81, 121, 20, 95, 256, 329, 112, 116, 37, 60, 10, 76, 353, 233, 248, 178, 276, 207, 28, 138, 232, 254, 234, 352, 264, 305, 85, 348, 184, 93, 104, 91, 61, 274, 58, 244, 140, 317, 287, 250, 230, 297, 129, 46, 224, 228, 156, 351, 144, 119, 31, 253, 3, 262, 141, 172, 239, 52, 150, 126, 359, 340, 36, 206, 321, 306, 117, 40, 364, 195, 54, 137, 246, 100, 82, 310, 229, 208, 135, 190, 181, 187, 318, 22, 56, 13, 334, 295, 336, 243, 320, 322, 379, 266, 209, 86, 332, 221, 50, 319, 88, 139, 183, 231, 335, 235, 149, 315, 293, 357, 196, 169, 105, 327, 79, 255, 182, 55, 337, 367, 131, 9, 51, 147, 301, 374, 382, 192, 343, 304, 344, 252, 90, 350, 23, 347, 308, 203, 238, 292, 179, 170, 279, 157, 261, 271, 107, 171, 219, 70, 69, 151, 49, 78, 349, 44, 102, 67, 35, 115, 371, 19, 30, 123, 218, 97, 87, 154, 283, 249, 63, 193, 17, 366, 161, 148, 124, 355, 71, 290, 153] + 54: [363, 71, 39, 80, 172, 341, 290, 311, 180, 228, 121, 70, 310, 357, 50, 325, 142, 109, 376, 186, 100, 67, 48, 85, 137, 62, 81, 262, 9, 175, 334, 110, 234, 367, 261, 89, 66, 270, 307, 134, 169, 297, 383, 106, 354, 336, 361, 378, 370, 299, 296, 83, 82, 250, 72, 254, 330, 119, 7, 329, 276, 333, 129, 64, 323, 190, 159, 279, 204, 237, 183, 318, 69, 163, 22, 242, 377, 273, 128, 266, 371, 219, 346, 29, 249, 108, 248, 374, 51, 65, 112, 256, 19, 200, 36, 148, 294, 231, 286, 253, 90, 260, 20, 324, 93, 57, 117, 86, 16, 132, 47, 32, 322, 91, 185, 272, 239, 184, 365, 302, 278, 264, 301, 136, 96, 33, 347, 191, 351, 12, 88, 145, 236, 274, 343, 362, 149, 35, 104, 252, 37, 342, 123, 13, 339, 223, 280, 380, 147, 257, 98, 306, 208, 305, 309, 3, 27, 156, 41, 196, 158, 17, 321, 182, 24, 21, 312, 49, 1, 366, 293, 107, 304, 173, 18, 300, 194, 229, 213, 6, 178, 233, 79, 60, 199, 116, 210, 188, 222, 349, 127, 160, 155, 356, 230, 153, 338, 150, 30, 281, 282, 221, 25, 314, 101, 332, 99, 218, 277, 225, 265, 92, 382, 157, 105, 135, 177, 373, 327, 122, 179, 162, 151, 316, 245, 243, 259, 235, 247, 331, 268, 46, 226, 154, 4, 77, 126, 78, 202, 232, 164, 11, 38, 337, 189, 216, 133, 174, 120, 187, 26, 118, 181, 375, 76, 197, 198, 335, 220, 152, 54, 381, 55, 368, 61, 144, 326, 111, 146, 167, 166, 23, 141, 251, 308, 59, 168, 289, 359, 102, 227, 292, 28, 138, 165, 193, 358, 320, 284, 258, 348, 295, 328, 287, 369, 53, 44, 345, 352, 372, 58, 10, 143, 113, 215, 45, 5, 217, 103, 140, 97, 315, 73, 171, 255, 313, 364, 353, 244, 43, 238, 271, 15, 224, 14, 31, 269, 203, 56, 360, 350, 291, 207, 340, 130, 205, 283, 42, 240, 95, 319, 267, 214, 8, 176, 87, 40, 75, 317, 288, 115, 161, 84, 124, 94, 34, 125, 170, 355, 131, 246, 298, 68, 209, 201, 114, 195, 275, 139, 192, 63, 74, 211, 206, 52, 263, 285, 303, 379, 2, 241, 344, 212, 0] + 55: [154, 323, 113, 256, 62, 39, 16, 41, 121, 106, 249, 71, 272, 300, 316, 233, 197, 276, 95, 260, 231, 343, 194, 136, 318, 157, 207, 313, 63, 160, 127, 173, 356, 321, 168, 315, 66, 68, 280, 169, 330, 362, 100, 345, 205, 222, 245, 109, 174, 90, 269, 43, 151, 89, 252, 353, 247, 40, 341, 176, 320, 164, 86, 239, 5, 325, 105, 378, 377, 47, 54, 241, 122, 254, 193, 22, 117, 232, 133, 208, 372, 336, 55, 147, 145, 261, 77, 376, 334, 75, 195, 351, 292, 144, 196, 215, 56, 80, 74, 326, 270, 309, 357, 307, 73, 78, 132, 223, 226, 60, 138, 26, 156, 112, 350, 190, 217, 236, 240, 228, 140, 346, 358, 278, 294, 302, 342, 188, 373, 359, 118, 38, 354, 374, 139, 203, 191, 155, 266, 158, 24, 25, 250, 204, 202, 116, 347, 115, 327, 235, 370, 227, 6, 70, 301, 134, 279, 44, 368, 271, 206, 163, 348, 375, 251, 380, 360, 187, 322, 257, 171, 52, 314, 214, 1, 365, 15, 335, 35, 243, 58, 107, 177, 369, 295, 175, 32, 108, 59, 379, 96, 317, 88, 340, 48, 76, 152, 4, 218, 192, 267, 363, 65, 99, 8, 331, 213, 0, 344, 46, 170, 289, 371, 246, 79, 53, 19, 119, 287, 101, 149, 82, 198, 103, 37, 30, 93, 209, 120, 125, 248, 162, 364, 21, 36, 288, 148, 167, 51, 34, 81, 308, 299, 29, 28, 355, 339, 311, 211, 273, 291, 200, 179, 349, 290, 383, 124, 17, 284, 91, 210, 265, 306, 286, 332, 277, 23, 84, 87, 264, 61, 367, 219, 253, 126, 104, 338, 150, 324, 183, 92, 262, 57, 142, 131, 9, 283, 69, 98, 146, 381, 130, 159, 7, 182, 111, 303, 382, 199, 293, 3, 333, 216, 366, 172, 259, 102, 274, 27, 224, 42, 135, 263, 184, 128, 85, 296, 31, 237, 281, 225, 268, 319, 137, 129, 275, 13, 166, 298, 181, 110, 220, 18, 229, 310, 305, 20, 11, 123, 180, 72, 234, 165, 238, 328, 304, 49, 2, 114, 12, 221, 312, 337, 189, 14, 352, 329, 361, 67, 178, 185, 10, 255, 94, 33, 153, 212, 258, 143, 97, 201, 297, 161, 50, 64, 242, 141, 282, 244, 83, 230, 45, 186, 285] + 56: [231, 340, 100, 199, 98, 371, 20, 333, 328, 316, 275, 66, 159, 150, 383, 0, 58, 40, 253, 135, 295, 166, 355, 110, 255, 270, 286, 115, 122, 134, 145, 347, 323, 274, 37, 16, 70, 93, 254, 272, 239, 261, 36, 25, 247, 142, 146, 3, 240, 225, 304, 167, 186, 158, 156, 89, 306, 46, 99, 18, 327, 245, 101, 223, 233, 376, 148, 10, 21, 79, 268, 280, 133, 164, 227, 368, 95, 94, 54, 341, 290, 200, 276, 372, 144, 130, 47, 337, 176, 349, 366, 111, 171, 301, 173, 90, 195, 15, 75, 343, 116, 184, 169, 236, 125, 161, 48, 2, 19, 288, 244, 97, 344, 334, 342, 278, 92, 252, 230, 121, 34, 153, 209, 297, 190, 348, 180, 314, 132, 293, 319, 65, 29, 86, 113, 131, 345, 203, 370, 377, 114, 237, 74, 85, 260, 213, 5, 311, 107, 296, 87, 196, 187, 6, 39, 219, 332, 14, 283, 309, 197, 73, 38, 137, 181, 31, 374, 331, 50, 33, 123, 282, 339, 299, 183, 262, 228, 28, 330, 289, 67, 352, 287, 157, 382, 212, 52, 7, 277, 91, 102, 143, 220, 82, 30, 8, 273, 88, 351, 356, 264, 162, 71, 292, 329, 381, 168, 120, 205, 27, 285, 250, 206, 198, 179, 12, 1, 17, 127, 56, 221, 201, 257, 84, 353, 124, 379, 367, 78, 43, 373, 258, 26, 325, 335, 216, 174, 165, 229, 208, 294, 178, 259, 177, 81, 163, 322, 249, 104, 117, 140, 302, 362, 154, 192, 72, 246, 300, 160, 193, 324, 338, 23, 22, 11, 13, 243, 364, 313, 360, 298, 119, 172, 307, 175, 57, 64, 350, 60, 152, 108, 358, 336, 217, 194, 235, 76, 224, 321, 44, 106, 279, 126, 53, 303, 9, 263, 308, 346, 41, 83, 77, 357, 118, 238, 103, 182, 312, 105, 305, 354, 317, 214, 151, 141, 129, 42, 62, 210, 218, 310, 4, 226, 139, 266, 361, 222, 204, 234, 215, 326, 45, 365, 318, 109, 315, 242, 265, 241, 147, 191, 155, 267, 189, 363, 251, 320, 284, 188, 80, 63, 207, 369, 359, 69, 281, 128, 136, 35, 269, 149, 138, 185, 248, 378, 271, 55, 96, 375, 112, 232, 291, 49, 170, 24, 211, 256, 68, 59, 32, 380, 51, 61, 202] + 57: [254, 125, 159, 121, 198, 257, 324, 267, 6, 283, 160, 163, 238, 91, 172, 213, 214, 113, 312, 111, 38, 55, 328, 226, 352, 209, 28, 132, 29, 237, 49, 211, 200, 168, 219, 256, 239, 78, 308, 258, 82, 221, 77, 153, 202, 262, 301, 304, 276, 148, 170, 342, 314, 7, 128, 53, 118, 150, 197, 88, 62, 272, 167, 373, 40, 51, 249, 356, 102, 203, 106, 95, 210, 196, 282, 155, 186, 162, 261, 315, 236, 255, 357, 333, 361, 370, 232, 10, 264, 206, 278, 227, 280, 367, 275, 343, 326, 205, 72, 93, 158, 63, 65, 124, 268, 350, 318, 79, 123, 182, 99, 253, 322, 228, 70, 83, 21, 183, 284, 13, 279, 171, 337, 90, 330, 18, 371, 97, 33, 59, 0, 32, 109, 194, 48, 364, 271, 266, 52, 327, 181, 15, 229, 372, 25, 187, 320, 11, 16, 368, 235, 137, 344, 169, 117, 64, 44, 354, 58, 39, 346, 73, 27, 298, 184, 147, 12, 380, 2, 164, 199, 362, 178, 215, 240, 145, 252, 24, 270, 126, 66, 36, 374, 139, 61, 115, 291, 292, 305, 34, 105, 376, 231, 246, 286, 157, 133, 156, 101, 274, 8, 204, 151, 22, 35, 251, 347, 161, 179, 297, 42, 46, 349, 92, 358, 112, 329, 313, 190, 321, 98, 334, 290, 247, 5, 265, 307, 281, 263, 50, 192, 76, 273, 218, 114, 340, 69, 245, 285, 56, 14, 89, 94, 154, 381, 57, 104, 359, 335, 300, 127, 317, 306, 80, 338, 316, 110, 9, 177, 277, 191, 54, 47, 119, 348, 71, 351, 369, 288, 17, 379, 363, 45, 303, 378, 87, 224, 242, 67, 103, 142, 244, 129, 189, 336, 201, 233, 293, 146, 165, 207, 302, 31, 220, 140, 223, 130, 60, 174, 345, 185, 75, 319, 43, 176, 341, 152, 20, 382, 116, 120, 225, 68, 136, 3, 296, 81, 311, 41, 332, 122, 135, 299, 241, 1, 222, 138, 107, 310, 100, 212, 193, 30, 37, 19, 86, 259, 295, 331, 74, 230, 85, 366, 108, 141, 217, 84, 180, 309, 250, 144, 365, 134, 243, 96, 173, 377, 208, 289, 325, 166, 149, 353, 23, 216, 323, 175, 26, 355, 248, 4, 188, 383, 260, 375, 287, 360, 339, 294, 269, 195, 131, 143, 234] + 58: [377, 169, 232, 324, 367, 132, 4, 190, 198, 229, 55, 239, 129, 236, 160, 183, 149, 362, 379, 334, 347, 113, 195, 8, 64, 243, 372, 330, 354, 374, 223, 109, 34, 15, 301, 91, 361, 279, 321, 168, 201, 340, 33, 38, 58, 134, 209, 308, 220, 114, 61, 27, 373, 9, 351, 256, 112, 376, 85, 42, 130, 162, 66, 185, 48, 202, 383, 284, 350, 326, 37, 0, 144, 148, 123, 345, 19, 35, 237, 170, 294, 87, 317, 41, 254, 247, 117, 145, 17, 30, 140, 89, 147, 292, 184, 204, 81, 194, 298, 76, 348, 101, 49, 67, 199, 368, 295, 342, 119, 104, 111, 307, 328, 380, 314, 207, 181, 238, 227, 346, 135, 172, 176, 288, 2, 287, 50, 133, 118, 5, 138, 221, 303, 103, 44, 349, 137, 52, 127, 218, 233, 158, 309, 280, 245, 98, 311, 263, 7, 344, 93, 69, 355, 116, 68, 6, 203, 315, 316, 20, 28, 281, 375, 82, 99, 215, 155, 382, 332, 13, 253, 161, 371, 293, 179, 136, 226, 24, 142, 359, 157, 336, 269, 120, 153, 78, 188, 323, 23, 320, 335, 364, 156, 92, 122, 248, 356, 210, 322, 22, 214, 178, 262, 47, 31, 12, 255, 197, 171, 1, 331, 143, 258, 306, 234, 378, 63, 302, 154, 107, 228, 265, 283, 94, 230, 296, 100, 54, 163, 146, 46, 369, 216, 88, 267, 343, 165, 80, 3, 370, 275, 205, 312, 249, 273, 211, 235, 72, 257, 36, 196, 141, 191, 97, 60, 242, 224, 192, 139, 59, 62, 53, 250, 240, 131, 208, 357, 278, 21, 11, 182, 105, 180, 102, 360, 86, 268, 200, 51, 96, 159, 193, 365, 363, 212, 164, 71, 10, 126, 305, 222, 77, 79, 271, 206, 358, 26, 40, 152, 341, 189, 277, 260, 352, 73, 187, 381, 244, 325, 83, 339, 75, 121, 300, 231, 65, 318, 115, 353, 219, 297, 213, 29, 128, 276, 290, 84, 39, 246, 286, 74, 217, 291, 166, 125, 108, 56, 366, 177, 151, 259, 14, 299, 289, 270, 175, 45, 18, 16, 272, 106, 274, 285, 329, 251, 25, 266, 90, 124, 333, 110, 241, 186, 327, 43, 70, 225, 173, 338, 261, 95, 264, 282, 337, 167, 57, 32, 304, 174, 310, 313, 252, 319, 150] + 59: [327, 220, 113, 105, 226, 374, 291, 172, 328, 151, 281, 290, 45, 40, 16, 181, 162, 315, 29, 47, 355, 348, 366, 6, 353, 199, 107, 333, 246, 34, 84, 330, 359, 44, 183, 53, 166, 303, 8, 141, 350, 375, 347, 345, 167, 4, 90, 189, 25, 260, 346, 182, 97, 251, 371, 150, 213, 295, 144, 196, 86, 176, 160, 279, 297, 248, 252, 245, 10, 236, 344, 238, 13, 228, 156, 48, 71, 27, 208, 134, 157, 93, 49, 24, 118, 296, 82, 165, 240, 14, 293, 87, 78, 72, 235, 154, 170, 202, 66, 33, 313, 70, 263, 130, 185, 324, 80, 255, 58, 212, 81, 35, 302, 163, 376, 338, 214, 158, 137, 92, 337, 123, 314, 32, 287, 41, 135, 184, 17, 326, 336, 275, 273, 307, 368, 319, 1, 56, 77, 171, 83, 99, 308, 215, 229, 195, 340, 73, 285, 5, 298, 288, 300, 186, 262, 382, 342, 109, 223, 254, 280, 334, 22, 60, 286, 258, 146, 369, 365, 2, 61, 194, 138, 43, 50, 124, 28, 74, 88, 249, 148, 102, 241, 237, 318, 360, 272, 244, 104, 122, 310, 271, 7, 94, 173, 332, 30, 147, 247, 331, 31, 372, 26, 266, 133, 175, 161, 12, 100, 169, 143, 19, 306, 69, 120, 354, 203, 51, 68, 299, 193, 63, 187, 217, 21, 89, 261, 115, 201, 230, 312, 108, 239, 192, 335, 378, 126, 323, 111, 268, 140, 370, 59, 234, 46, 95, 381, 294, 18, 9, 23, 243, 112, 91, 57, 127, 267, 383, 20, 218, 356, 179, 180, 177, 36, 101, 242, 0, 206, 64, 136, 253, 207, 153, 132, 210, 357, 103, 358, 98, 316, 305, 39, 264, 55, 190, 15, 341, 209, 379, 200, 65, 75, 233, 311, 76, 42, 38, 178, 309, 114, 278, 85, 159, 364, 320, 373, 142, 329, 339, 198, 361, 155, 131, 274, 362, 380, 110, 325, 231, 67, 145, 277, 191, 321, 284, 216, 257, 276, 219, 121, 367, 139, 116, 117, 79, 224, 250, 363, 352, 256, 197, 204, 221, 265, 377, 317, 232, 128, 164, 304, 168, 211, 62, 225, 174, 54, 222, 269, 259, 289, 349, 96, 292, 227, 149, 129, 322, 343, 37, 3, 106, 11, 119, 283, 301, 282, 125, 270, 205, 152, 52, 351, 188] + 60: [313, 94, 195, 372, 78, 65, 208, 239, 49, 166, 98, 272, 59, 27, 169, 151, 20, 6, 240, 332, 110, 215, 116, 186, 90, 85, 146, 182, 95, 219, 225, 338, 282, 361, 328, 258, 145, 356, 291, 72, 43, 289, 340, 10, 218, 214, 288, 201, 124, 174, 335, 129, 339, 230, 185, 74, 294, 180, 329, 278, 231, 238, 99, 336, 255, 269, 183, 11, 237, 274, 25, 134, 142, 293, 196, 300, 322, 256, 64, 197, 168, 13, 209, 179, 42, 204, 96, 86, 135, 367, 32, 44, 242, 154, 375, 355, 351, 83, 310, 170, 244, 212, 312, 81, 34, 319, 119, 15, 131, 161, 241, 253, 0, 277, 37, 1, 188, 264, 377, 304, 21, 107, 66, 7, 192, 370, 286, 18, 325, 270, 45, 228, 216, 50, 68, 97, 159, 254, 163, 4, 101, 308, 317, 22, 123, 60, 265, 128, 303, 316, 247, 297, 205, 76, 365, 345, 245, 143, 263, 359, 84, 283, 132, 318, 80, 267, 57, 342, 233, 70, 187, 30, 369, 314, 380, 148, 363, 376, 127, 287, 261, 224, 160, 147, 165, 281, 164, 103, 118, 89, 62, 299, 257, 139, 40, 121, 243, 113, 194, 334, 2, 108, 353, 193, 213, 117, 235, 246, 75, 58, 221, 36, 344, 9, 3, 279, 26, 346, 31, 200, 100, 357, 276, 12, 315, 307, 262, 330, 19, 198, 374, 378, 301, 306, 292, 177, 73, 77, 210, 207, 16, 47, 199, 236, 266, 39, 167, 302, 28, 35, 290, 311, 206, 184, 271, 364, 152, 69, 114, 51, 92, 162, 38, 8, 323, 52, 273, 33, 190, 106, 296, 14, 125, 105, 202, 252, 320, 280, 379, 29, 175, 326, 173, 138, 352, 87, 373, 348, 268, 220, 126, 284, 260, 343, 48, 153, 181, 327, 234, 112, 354, 232, 203, 55, 172, 189, 248, 93, 217, 88, 382, 176, 56, 141, 360, 91, 149, 341, 349, 144, 331, 383, 67, 23, 178, 137, 41, 211, 111, 191, 350, 71, 275, 321, 136, 157, 381, 158, 133, 102, 115, 347, 222, 120, 155, 366, 337, 46, 309, 122, 223, 82, 140, 150, 362, 358, 259, 24, 5, 61, 305, 171, 53, 285, 250, 130, 79, 368, 371, 333, 156, 251, 324, 54, 104, 229, 298, 249, 17, 227, 295, 109, 226, 63] + 61: [221, 248, 107, 360, 247, 241, 363, 330, 278, 235, 129, 115, 159, 32, 43, 348, 233, 44, 190, 47, 336, 207, 202, 139, 292, 8, 365, 225, 67, 71, 342, 193, 311, 163, 364, 312, 13, 314, 223, 236, 56, 281, 134, 41, 16, 356, 94, 58, 127, 167, 88, 120, 326, 270, 45, 12, 51, 237, 112, 110, 15, 169, 337, 224, 77, 83, 82, 109, 206, 145, 33, 72, 78, 275, 125, 269, 295, 124, 204, 298, 91, 239, 101, 74, 201, 272, 138, 250, 60, 226, 210, 50, 40, 128, 319, 0, 179, 100, 161, 230, 170, 323, 59, 253, 372, 142, 119, 333, 42, 156, 75, 258, 297, 29, 27, 379, 308, 368, 200, 259, 38, 273, 37, 126, 240, 317, 332, 152, 274, 25, 255, 358, 24, 293, 28, 339, 220, 285, 131, 267, 325, 296, 343, 208, 338, 36, 307, 172, 157, 178, 209, 103, 17, 215, 66, 57, 188, 243, 133, 361, 65, 84, 166, 11, 257, 23, 147, 173, 383, 214, 9, 79, 196, 132, 194, 106, 244, 271, 6, 99, 262, 305, 212, 105, 347, 377, 175, 183, 320, 229, 189, 53, 61, 381, 324, 86, 335, 205, 10, 184, 353, 238, 55, 327, 34, 349, 309, 228, 20, 316, 300, 113, 2, 260, 328, 290, 154, 137, 280, 242, 283, 340, 199, 359, 70, 130, 354, 123, 251, 185, 373, 265, 35, 222, 279, 289, 367, 334, 350, 155, 73, 197, 344, 231, 5, 351, 19, 1, 195, 371, 143, 288, 168, 286, 227, 261, 191, 268, 366, 313, 304, 3, 52, 102, 186, 153, 306, 341, 85, 141, 176, 14, 162, 352, 187, 18, 7, 111, 164, 48, 256, 64, 277, 382, 76, 217, 96, 321, 93, 46, 263, 216, 108, 165, 203, 264, 218, 294, 213, 192, 369, 31, 282, 87, 362, 90, 114, 252, 329, 69, 104, 249, 322, 89, 375, 22, 219, 81, 144, 284, 370, 345, 54, 301, 49, 331, 180, 146, 140, 276, 151, 355, 376, 266, 171, 287, 211, 299, 315, 303, 136, 122, 117, 380, 21, 68, 198, 291, 177, 181, 160, 92, 118, 150, 135, 121, 374, 245, 97, 158, 378, 346, 30, 174, 80, 26, 4, 98, 148, 63, 310, 234, 232, 246, 95, 182, 318, 302, 254, 39, 62, 357, 116, 149] +layer_updates_per_iter: 0 +num_slots: 384 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b1024-c512x1024-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b1024-c512x1024-mtp.yaml new file mode 100644 index 0000000000..a7d9f09923 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b1024-c512x1024-mtp.yaml @@ -0,0 +1,132 @@ +# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml +# base + zip_override_mtp_maxtpt[0]): 1p1d max-throughput (dep4 prefill / dep8 decode, mem-fraction 0.75). +# One flat YAML per concrete topology, matching the 8k1k local recipe layout +# (//-//.yaml). + +schema: 2 +name: b200-fp4-mtp-max-tpt-dep4-1p-dep8-1d +model: + path: dsr1 + container: dynamo-sglang + precision: fp4 + +dynamo: + install: true + + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 1024 + disable-cuda-graph: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + decode: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_ENABLE_SPEC_V2: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.75 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + req_rate: inf + concurrencies: 512x1024 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p2d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b512-c512-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p2d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b512-c512-mtp.yaml new file mode 100644 index 0000000000..8400914132 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p2d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b512-c512-mtp.yaml @@ -0,0 +1,132 @@ +# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml +# base + zip_override_mtp_maxtpt[1]): 1p2d max-throughput (dep4 prefill / dep8 decode, mem-fraction 0.85). +# One flat YAML per concrete topology, matching the 8k1k local recipe layout +# (//-//.yaml). + +schema: 2 +name: b200-fp4-mtp-max-tpt-dep4-1p-dep8-2d +model: + path: dsr1 + container: dynamo-sglang + precision: fp4 + +dynamo: + install: true + + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + disable-cuda-graph: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + decode: + nodes: 2 + workers: 2 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_ENABLE_SPEC_V2: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + cuda-graph-max-bs: 512 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + req_rate: inf + concurrencies: '512' + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p5d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c16x512-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p5d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c16x512-mtp.yaml new file mode 100644 index 0000000000..21d89470d9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p5d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c16x512-mtp.yaml @@ -0,0 +1,127 @@ +# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml +# base + zip_override_mtp_lowlat[0]): 1p5d low-latency (dep4 prefill / tep8 decode, 5 decode nodes). +# One flat YAML per concrete topology, matching the 8k1k local recipe layout +# (//-//.yaml). + +schema: 2 +name: b200-fp4-mtp-low-latency-dep4-1p-tep8-5d +model: + path: dsr1 + container: dynamo-sglang + precision: fp4 + +dynamo: + install: true + + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + disable-cuda-graph: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + decode: + nodes: 5 + workers: 5 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + cuda-graph-max-bs: 512 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + req_rate: inf + concurrencies: 16x512 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c32x64x256x512-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c32x64x256x512-mtp.yaml new file mode 100644 index 0000000000..33819f802e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c32x64x256x512-mtp.yaml @@ -0,0 +1,127 @@ +# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml +# base + zip_override_mtp_lowlat[1]): 1p6d low-latency (dep4 prefill / tep8 decode, 6 decode nodes). +# One flat YAML per concrete topology, matching the 8k1k local recipe layout +# (//-//.yaml). + +schema: 2 +name: b200-fp4-mtp-low-latency-dep4-1p-tep8-6d +model: + path: dsr1 + container: dynamo-sglang + precision: fp4 + +dynamo: + install: true + + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + disable-cuda-graph: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + decode: + nodes: 6 + workers: 6 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + cuda-graph-max-bs: 512 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + req_rate: inf + concurrencies: 32x64x256x512 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp4-d-tp8-b512-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp4-d-tp8-b512-c32-mtp.yaml new file mode 100644 index 0000000000..a3af5ab25a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp4-d-tp8-b512-c32-mtp.yaml @@ -0,0 +1,144 @@ +schema: 2 +name: "b200-fp4-mtp-low-latency-1p-tp4-1d-tp8" + +# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml +# base + zip_override_mtp_lowlat[2], 1p-tp4 prefill / 1d-tp8 decode). +# One flat YAML per concrete topology, matching the InferenceX glm5 disagg +# layout (//-//.yaml). + +dynamo: + install: true + + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 3 + nginx_container: nginx-sqsh + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.12.post1" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + args: + disaggregation-mode: "prefill" + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.95 + max-total-tokens: 32768 + chunked-prefill-size: 24576 + cuda-graph-max-bs: 256 + max-running-requests: 512 + scheduler-recv-interval: 10 + load-balance-method: "round_robin" + disaggregation-bootstrap-port: 30001 + data-parallel-size: 1 + tensor-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: false + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + decode: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + args: + disaggregation-mode: "decode" + served-model-name: "deepseek-ai/DeepSeek-R1" + prefill-round-robin-balance: true + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + disaggregation-bootstrap-port: 30001 + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.85 + chunked-prefill-size: 8192 + cuda-graph-max-bs: 128 + max-running-requests: 512 + scheduler-recv-interval: 10 + tensor-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: false + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: 300 + concurrencies: "32" + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c512-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c512-mtp.yaml new file mode 100644 index 0000000000..a556fcc602 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c512-mtp.yaml @@ -0,0 +1,147 @@ +schema: 2 +name: "b200-fp4-mtp2-throughput-1p-dep4-1d-dep8" + +# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml +# base + override_mtp2_throughput_1p1d, DEP4 prefill / DEP8 decode, MTP2). +# One flat YAML per concrete topology, matching the InferenceX glm5 disagg +# layout (//-//.yaml). + +dynamo: + install: true + + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 3 + nginx_container: nginx-sqsh + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.12.post1" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + args: + disaggregation-mode: "prefill" + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.95 + max-total-tokens: 32768 + chunked-prefill-size: 24576 + cuda-graph-max-bs: 256 + max-running-requests: 2048 + scheduler-recv-interval: 1 + load-balance-method: "round_robin" + disaggregation-bootstrap-port: + data-parallel-size: 4 + tensor-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + + decode: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + args: + disaggregation-mode: "decode" + served-model-name: "deepseek-ai/DeepSeek-R1" + prefill-round-robin-balance: true + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + disaggregation-bootstrap-port: 30001 + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.85 + chunked-prefill-size: 8192 + cuda-graph-max-bs: 1024 + max-running-requests: 2048 + scheduler-recv-interval: 1 + data-parallel-size: 8 + tensor-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: 300 + concurrencies: "512" + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p3d-p-tp4-d-tp8-b512-c32x64-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p3d-p-tp4-d-tp8-b512-c32x64-mtp.yaml new file mode 100644 index 0000000000..6c79cf8355 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p3d-p-tp4-d-tp8-b512-c32x64-mtp.yaml @@ -0,0 +1,144 @@ +schema: 2 +name: "b200-fp4-mtp-low-latency-1p-tp4-3d-tp8" + +# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml +# base + zip_override_mtp_lowlat[1], 1p-tp4 prefill / 3d-tp8 decode). +# One flat YAML per concrete topology, matching the InferenceX glm5 disagg +# layout (//-//.yaml). + +dynamo: + install: true + + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 3 + nginx_container: nginx-sqsh + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.12.post1" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + args: + disaggregation-mode: "prefill" + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.95 + max-total-tokens: 32768 + chunked-prefill-size: 24576 + cuda-graph-max-bs: 256 + max-running-requests: 512 + scheduler-recv-interval: 10 + load-balance-method: "round_robin" + disaggregation-bootstrap-port: 30001 + data-parallel-size: 1 + tensor-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: false + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + decode: + nodes: 3 + workers: 3 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + args: + disaggregation-mode: "decode" + served-model-name: "deepseek-ai/DeepSeek-R1" + prefill-round-robin-balance: true + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + disaggregation-bootstrap-port: 30001 + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.85 + chunked-prefill-size: 8192 + cuda-graph-max-bs: 128 + max-running-requests: 512 + scheduler-recv-interval: 10 + tensor-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: false + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: 300 + concurrencies: "32x64" + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p5d-p-tp4-d-tp8-b512-c4x8x16x32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p5d-p-tp4-d-tp8-b512-c4x8x16x32-mtp.yaml new file mode 100644 index 0000000000..6a16ed52ee --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p5d-p-tp4-d-tp8-b512-c4x8x16x32-mtp.yaml @@ -0,0 +1,144 @@ +schema: 2 +name: "b200-fp4-mtp-low-latency-1p-tp4-5d-tp8" + +# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml +# base + zip_override_mtp_lowlat[0], 1p-tp4 prefill / 5d-tp8 decode). +# One flat YAML per concrete topology, matching the InferenceX glm5 disagg +# layout (//-//.yaml). + +dynamo: + install: true + + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 3 + nginx_container: nginx-sqsh + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.12.post1" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + args: + disaggregation-mode: "prefill" + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.95 + max-total-tokens: 32768 + chunked-prefill-size: 24576 + cuda-graph-max-bs: 256 + max-running-requests: 512 + scheduler-recv-interval: 10 + load-balance-method: "round_robin" + disaggregation-bootstrap-port: 30001 + data-parallel-size: 1 + tensor-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: false + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + decode: + nodes: 5 + workers: 5 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + args: + disaggregation-mode: "decode" + served-model-name: "deepseek-ai/DeepSeek-R1" + prefill-round-robin-balance: true + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + disaggregation-bootstrap-port: 30001 + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.85 + chunked-prefill-size: 8192 + cuda-graph-max-bs: 128 + max-running-requests: 512 + scheduler-recv-interval: 10 + tensor-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: false + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: 300 + concurrencies: "4x8x16x32" + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-2p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c768-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-2p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c768-mtp.yaml new file mode 100644 index 0000000000..f5b7b4b4bf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-2p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c768-mtp.yaml @@ -0,0 +1,147 @@ +schema: 2 +name: "b200-fp4-mtp2-throughput-2p-dep4-1d-dep8" + +# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml +# base + override_mtp2_throughput_2p1d, DEP4 prefill / DEP8 decode, MTP2). +# One flat YAML per concrete topology, matching the InferenceX glm5 disagg +# layout (//-//.yaml). + +dynamo: + install: true + + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 3 + nginx_container: nginx-sqsh + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.12.post1" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + args: + disaggregation-mode: "prefill" + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.95 + max-total-tokens: 32768 + chunked-prefill-size: 24576 + cuda-graph-max-bs: 256 + max-running-requests: 2048 + scheduler-recv-interval: 1 + load-balance-method: "round_robin" + disaggregation-bootstrap-port: + data-parallel-size: 4 + tensor-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + + decode: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + args: + disaggregation-mode: "decode" + served-model-name: "deepseek-ai/DeepSeek-R1" + prefill-round-robin-balance: true + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + disaggregation-bootstrap-port: 30001 + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.85 + chunked-prefill-size: 8192 + cuda-graph-max-bs: 1024 + max-running-requests: 2048 + scheduler-recv-interval: 1 + data-parallel-size: 8 + tensor-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: 300 + concurrencies: "768" + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c1024-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c1024-mtp.yaml new file mode 100644 index 0000000000..31f8450c18 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c1024-mtp.yaml @@ -0,0 +1,147 @@ +schema: 2 +name: "b200-fp4-mtp2-throughput-3p-dep4-1d-dep8" + +# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml +# base + override_mtp2_throughput_3p1d, DEP4 prefill / DEP8 decode, MTP2). +# One flat YAML per concrete topology, matching the InferenceX glm5 disagg +# layout (//-//.yaml). + +dynamo: + install: true + + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 3 + nginx_container: nginx-sqsh + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.12.post1" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 3 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + args: + disaggregation-mode: "prefill" + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.95 + max-total-tokens: 32768 + chunked-prefill-size: 24576 + cuda-graph-max-bs: 256 + max-running-requests: 2048 + scheduler-recv-interval: 1 + load-balance-method: "round_robin" + disaggregation-bootstrap-port: + data-parallel-size: 4 + tensor-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + + decode: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + args: + disaggregation-mode: "decode" + served-model-name: "deepseek-ai/DeepSeek-R1" + prefill-round-robin-balance: true + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + disaggregation-bootstrap-port: 30001 + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.85 + chunked-prefill-size: 8192 + cuda-graph-max-bs: 1024 + max-running-requests: 2048 + scheduler-recv-interval: 1 + data-parallel-size: 8 + tensor-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: 300 + concurrencies: "1024" + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c512-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c512-mtp.yaml new file mode 100644 index 0000000000..1ee3d2316e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c512-mtp.yaml @@ -0,0 +1,147 @@ +schema: 2 +name: "b200-fp4-mtp2-throughput-4p-dep4-1d-dep8" + +# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml +# base + override_mtp2_throughput_4p1d, DEP4 prefill / DEP8 decode, MTP2). +# One flat YAML per concrete topology, matching the InferenceX glm5 disagg +# layout (//-//.yaml). + +dynamo: + install: true + + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 3 + nginx_container: nginx-sqsh + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.12.post1" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 4 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + args: + disaggregation-mode: "prefill" + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.95 + max-total-tokens: 32768 + chunked-prefill-size: 24576 + cuda-graph-max-bs: 256 + max-running-requests: 2048 + scheduler-recv-interval: 1 + load-balance-method: "round_robin" + disaggregation-bootstrap-port: + data-parallel-size: 4 + tensor-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + + decode: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + args: + disaggregation-mode: "decode" + served-model-name: "deepseek-ai/DeepSeek-R1" + prefill-round-robin-balance: true + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + disaggregation-bootstrap-port: 30001 + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.85 + chunked-prefill-size: 8192 + cuda-graph-max-bs: 1024 + max-running-requests: 2048 + scheduler-recv-interval: 1 + data-parallel-size: 8 + tensor-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: 300 + concurrencies: "512" + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-5p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c2048-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-5p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c2048-mtp.yaml new file mode 100644 index 0000000000..3ae4675ab4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-5p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c2048-mtp.yaml @@ -0,0 +1,147 @@ +schema: 2 +name: "b200-fp4-mtp2-throughput-5p-dep4-1d-dep8" + +# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml +# base + override_mtp2_throughput_5p1d, DEP4 prefill / DEP8 decode, MTP2). +# One flat YAML per concrete topology, matching the InferenceX glm5 disagg +# layout (//-//.yaml). + +dynamo: + install: true + + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 3 + nginx_container: nginx-sqsh + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.12.post1" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 3 + workers: 5 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + args: + disaggregation-mode: "prefill" + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.95 + max-total-tokens: 32768 + chunked-prefill-size: 24576 + cuda-graph-max-bs: 256 + max-running-requests: 2048 + scheduler-recv-interval: 1 + load-balance-method: "round_robin" + disaggregation-bootstrap-port: + data-parallel-size: 4 + tensor-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + + decode: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + args: + disaggregation-mode: "decode" + served-model-name: "deepseek-ai/DeepSeek-R1" + prefill-round-robin-balance: true + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + disaggregation-bootstrap-port: 30001 + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.85 + chunked-prefill-size: 8192 + cuda-graph-max-bs: 1024 + max-running-requests: 2048 + scheduler-recv-interval: 1 + data-parallel-size: 8 + tensor-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: 300 + concurrencies: "2048" + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-mtp-4p1d-c2048.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-mtp-4p1d-c2048.yaml new file mode 100644 index 0000000000..69f86ed623 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-mtp-4p1d-c2048.yaml @@ -0,0 +1,123 @@ +# DeepSeek-R1-0528 FP4 8k/1k B200 dynamo-sglang disagg — MTP, 4P:1D (c2048) +# Topology: 4 prefill workers (TP4/EP4 dp-attn) + 1 decode worker (TP8/EP8 dp-attn) = 24 GPU. +# Dedicated high-concurrency tune: +# SGLANG_HACK_SEQ_BOOTSTRAP_ROOM performance knob (not output-affecting; standard in GB200 recipes). +# max-running-requests raised to 2048 to avoid admission throttle at high concurrency. +# scheduler-recv-interval: 1 on decode. +# symm-mem + allreduce fusion retained. +# prefill stream-interval: 100; decode stream-interval: 34 (tuned for the c2048 batch cadence). +schema: 2 +name: b200-fp4-dsr1_8k1k_mtp_4p1d_c2048 +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.12.post1" + precision: fp4 +resources: + gpu_type: b200 + gpus_per_node: 8 +frontend: + type: dynamo +dynamo: + install: true + source: + rev: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" +engine: sglang +roles: + prefill: + nodes: 2 + workers: 4 + gpus: 4 + env: &env + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: prefill + load-balance-method: round_robin + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 9600 + max-running-requests: 2048 + disable-cuda-graph: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + enable-flashinfer-allreduce-fusion: true + enable-symm-mem: true + disable-radix-cache: true + stream-interval: 100 + watchdog-timeout: 1000000 + decode: + nodes: 1 + workers: 1 + env: *env + args: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: decode + prefill-round-robin-balance: true + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.75 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 9600 + max-running-requests: 2048 + cuda-graph-max-bs: 1024 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + enable-flashinfer-allreduce-fusion: true + enable-symm-mem: true + disable-radix-cache: true + stream-interval: 34 + scheduler-recv-interval: 1 + watchdog-timeout: 1000000 + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '2048' + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml new file mode 100644 index 0000000000..f37a816d6e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml @@ -0,0 +1,369 @@ +# B200-FP4 8k1k — STP and MTP in one file +# +# Three modes distinguished by override key names: +# override_stp_tp4 / override_mtp_tp4: TP4 prefill (DP=1, EP=1) — low-latency single-node +# zip_override_stp_lowlat / zip_override_mtp_lowlat: dep4 prefill + tep8 decode (DP=1) +# override_stp_maxtpt_7p2d / override_mtp_maxtpt_7p2d: dep4 prefill + dep8 decode, 7p2d +# override_mtp_maxtpt_4p1d: MTP-only 4p1d, no frontends, env-var FP4 backend +# +# Usage: +# srtctl apply -f recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml # all 11 variants +# srtctl apply -f recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml:*stp* # all STP variants +# srtctl apply -f recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml:*mtp* # all MTP variants +# srtctl apply -f recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml:override_stp_tp4 # STP tp4 only +# srtctl apply -f recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml:zip_override_stp_lowlat[0] # STP 1p1d only +# srtctl dry-run -f recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml # preview + +schema: 2 +base: + name: "b200-fp4-stp-8k1k" + + dynamo: + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + source: + pypi: 0.8.1 + + model: + path: "dsr1" + container: "dynamo-sglang" + precision: "fp4" + + frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + + resources: + gpu_type: "b200" + gpus_per_node: 8 + + engine: sglang + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: "1" + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + quantization: "modelopt_fp4" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 9600 + max-running-requests: 512 + disable-cuda-graph: true + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + # Attention + attention-backend: "trtllm_mla" + kv-cache-dtype: "fp8_e4m3" + + # MoE + moe-runner-backend: "flashinfer_trtllm" + moe-dense-tp-size: 1 + fp4-gemm-backend: "flashinfer_trtllm" + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + + decode: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: "1" + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + quantization: "modelopt_fp4" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 9600 + max-running-requests: 512 + cuda-graph-max-bs: 512 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + + # Attention + attention-backend: "trtllm_mla" + kv-cache-dtype: "fp8_e4m3" + + # MoE + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_trtllm" + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + + health_check: + max_attempts: 360 + interval_seconds: 10 + + benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + + +# STP TP4 prefill mode: TP4 (DP=1, EP=1) instead of dep4 — low-latency single-node +override_stp_tp4: + name: "b200-fp4-stp-low-latency-tp4-1p-tp8-1d" + frontend: + num_additional_frontends: 2 + roles: + prefill: + args: + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: + enable-dp-lm-head: + decode: + args: + expert-parallel-size: 1 + benchmark: + concurrencies: "4x8x16x64" + + +# MTP TP4 prefill mode: same as STP tp4 but adds EAGLE speculative decoding +override_mtp_tp4: + name: "b200-fp4-mtp-low-latency-tp4-1p-tp8-1d" + frontend: + num_additional_frontends: 2 + roles: + prefill: + env: + SGLANG_ENABLE_SPEC_V2: "1" + args: + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: + enable-dp-lm-head: + decode: + env: + SGLANG_ENABLE_SPEC_V2: "1" + args: + expert-parallel-size: 1 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + benchmark: + concurrencies: "4x8x16x64" + + +# STP low-latency: dep4 prefill + tep8 decode (DP=1), scale sweep 1p1d/1p5d/2p5d +zip_override_stp_lowlat: + name: + - "b200-fp4-stp-low-latency-dep4-1p-tep8-1d" + - "b200-fp4-stp-low-latency-dep4-1p-tep8-5d" + - "b200-fp4-stp-low-latency-dep4-2p-tep8-5d" + resources: {} + roles: + prefill: + nodes: [1, 1, 2] + workers: [1, 1, 2] + decode: + nodes: [1, 5, 5] + workers: [1, 5, 5] + benchmark: + concurrencies: ["64x128", "8", "4x128"] + + +# MTP low-latency: same scales as STP, adds EAGLE speculative decoding +zip_override_mtp_lowlat: + name: + - "b200-fp4-mtp-low-latency-dep4-1p-tep8-1d" + - "b200-fp4-mtp-low-latency-dep4-1p-tep8-5d" + - "b200-fp4-mtp-low-latency-dep4-2p-tep8-5d" + resources: {} + roles: + prefill: + nodes: [1, 1, 2] + workers: [1, 1, 2] + env: + SGLANG_ENABLE_SPEC_V2: "1" + decode: + nodes: [1, 5, 5] + workers: [1, 5, 5] + env: + SGLANG_ENABLE_SPEC_V2: "1" + args: + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + benchmark: + concurrencies: ["64x128", "8", "4x128"] + + +# STP max-throughput 7p2d: dep4 prefill + dep8 decode, flashinfer_cutlass backend +override_stp_maxtpt_7p2d: + name: "b200-fp4-stp-max-tpt-dep4-7p-dep8-2d" + resources: {} + roles: + prefill: + nodes: 7 + workers: 7 + args: + max-prefill-tokens: 65536 + chunked-prefill-size: 65536 + max-running-requests: 1024 + fp4-gemm-backend: "flashinfer_cutlass" + decode: + nodes: 2 + workers: 2 + env: + SGLANG_MOE_NVFP4_DISPATCH: "1" + args: + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + max-running-requests: 2048 + cuda-graph-max-bs: 1024 + fp4-gemm-backend: "flashinfer_cutlass" + benchmark: + concurrencies: "1024x2048" + + +# MTP max-throughput 7p2d: same as STP but adds EAGLE speculative decoding +override_mtp_maxtpt_7p2d: + name: "b200-fp4-mtp-max-tpt-dep4-7p-dep8-2d" + resources: {} + roles: + prefill: + nodes: 7 + workers: 7 + env: + SGLANG_ENABLE_SPEC_V2: "1" + args: + max-prefill-tokens: 65536 + chunked-prefill-size: 65536 + max-running-requests: 1024 + fp4-gemm-backend: "flashinfer_cutlass" + decode: + nodes: 2 + workers: 2 + env: + SGLANG_MOE_NVFP4_DISPATCH: "1" + SGLANG_ENABLE_SPEC_V2: "1" + args: + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + max-running-requests: 2048 + cuda-graph-max-bs: 1024 + fp4-gemm-backend: "flashinfer_cutlass" + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + benchmark: + concurrencies: "1024x2048" + + +# MTP-only: 4p1d, no frontends, SGLANG_FLASHINFER_FP4_GEMM_BACKEND env var (fp4-gemm-backend: null +# removes the sglang_config key), mem-fraction=0.75 for decode +override_mtp_maxtpt_4p1d: + name: "b200-fp4-mtp-max-tpt-dep4-4p-dep8-1d" + dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + source: + pypi: "0.8.0" + frontend: + resources: {} + roles: + prefill: + nodes: 4 + workers: 4 + env: + SGLANG_ENABLE_SPEC_V2: "1" + args: + max-running-requests: 1024 + fp4-gemm-backend: + decode: + nodes: 1 + workers: 1 + env: + SGLANG_MOE_NVFP4_DISPATCH: "1" + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" + SGLANG_ENABLE_SPEC_V2: "1" + args: + mem-fraction-static: 0.75 + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + fp4-gemm-backend: + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + benchmark: + concurrencies: "1024" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p1d-p-tp8-dp8-d-tp8-ep8-dp8-b256-c160x288-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p1d-p-tp8-dp8-d-tp8-ep8-dp8-b256-c160x288-mtp.yaml new file mode 100644 index 0000000000..a7f22c418a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p1d-p-tp8-dp8-d-tp8-ep8-dp8-b256-c160x288-mtp.yaml @@ -0,0 +1,148 @@ +schema: 2 +name: b200-fp8-mtp-max-tpt-dep8-1p-2d + +dynamo: + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + source: + pypi: 0.9.1 + +model: + path: dsr1-fp8 + container: dynamo-sglang + precision: fp8 + +resources: + gpu_type: b200 + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + SGLANG_ENABLE_SPEC_V2: '1' + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + load-balance-method: round_robin + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 8192 + chunked-prefill-size: 65536 + max-running-requests: 8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + + decode: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + SGLANG_ENABLE_SPEC_V2: '1' + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + context-length: 9600 + max-running-requests: 256 + cuda-graph-max-bs: 32 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 720 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + concurrencies: 160x288 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p1d-p-tp8-dp8-d-tp8-ep8-dp8-b256-c160x288-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p1d-p-tp8-dp8-d-tp8-ep8-dp8-b256-c160x288-stp.yaml new file mode 100644 index 0000000000..54f90f4235 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p1d-p-tp8-dp8-d-tp8-ep8-dp8-b256-c160x288-stp.yaml @@ -0,0 +1,144 @@ +schema: 2 +name: b200-fp8-stp-max-tpt-dep8-1p-1d + +dynamo: + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + source: + pypi: 0.9.1 + +model: + path: dsr1-fp8 + container: dynamo-sglang + precision: fp8 + +resources: + gpu_type: b200 + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + load-balance-method: round_robin + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 8192 + chunked-prefill-size: 65536 + max-running-requests: 8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + + decode: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + context-length: 9600 + max-running-requests: 256 + cuda-graph-max-bs: 256 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + concurrencies: 160x288 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p2d-p-tp8-dp8-d-tp8-ep8-dp8-b128-c288-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p2d-p-tp8-dp8-d-tp8-ep8-dp8-b128-c288-mtp.yaml new file mode 100644 index 0000000000..55f328afa4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p2d-p-tp8-dp8-d-tp8-ep8-dp8-b128-c288-mtp.yaml @@ -0,0 +1,148 @@ +schema: 2 +name: b200-fp8-mtp-max-tpt-dep8-1p-1d + +dynamo: + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + source: + pypi: 0.9.1 + +model: + path: dsr1-fp8 + container: dynamo-sglang + precision: fp8 + +resources: + gpu_type: b200 + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + SGLANG_ENABLE_SPEC_V2: '1' + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + load-balance-method: round_robin + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 8192 + chunked-prefill-size: 65536 + max-running-requests: 8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + + decode: + nodes: 2 + workers: 2 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + SGLANG_ENABLE_SPEC_V2: '1' + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + context-length: 9600 + max-running-requests: 128 + cuda-graph-max-bs: 16 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 720 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + concurrencies: '288' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p2d-p-tp8-dp8-d-tp8-ep8-dp8-b128-c288-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p2d-p-tp8-dp8-d-tp8-ep8-dp8-b128-c288-stp.yaml new file mode 100644 index 0000000000..2a55518c07 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p2d-p-tp8-dp8-d-tp8-ep8-dp8-b128-c288-stp.yaml @@ -0,0 +1,144 @@ +schema: 2 +name: b200-fp8-stp-max-tpt-dep8-1p-2d + +dynamo: + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + source: + pypi: 0.9.1 + +model: + path: dsr1-fp8 + container: dynamo-sglang + precision: fp8 + +resources: + gpu_type: b200 + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + load-balance-method: round_robin + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 8192 + chunked-prefill-size: 65536 + max-running-requests: 8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + + decode: + nodes: 2 + workers: 2 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + context-length: 9600 + max-running-requests: 128 + cuda-graph-max-bs: 128 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + concurrencies: '288' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p3d-p-tp8-dp8-d-tp8-b32-c128-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p3d-p-tp8-dp8-d-tp8-b32-c128-mtp.yaml new file mode 100644 index 0000000000..8544eea9db --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p3d-p-tp8-dp8-d-tp8-b32-c128-mtp.yaml @@ -0,0 +1,145 @@ +schema: 2 +name: b200-fp8-mtp-low-latency-tep8-1p-1d + +dynamo: + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + source: + pypi: 0.9.1 + +model: + path: dsr1-fp8 + container: dynamo-sglang + precision: fp8 + +resources: + gpu_type: b200 + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + SGLANG_ENABLE_SPEC_V2: '1' + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + load-balance-method: round_robin + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 8192 + chunked-prefill-size: 65536 + max-running-requests: 8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + + decode: + nodes: 3 + workers: 3 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + SGLANG_ENABLE_SPEC_V2: '1' + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + context-length: 9600 + max-running-requests: 32 + cuda-graph-max-bs: 32 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 720 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + concurrencies: '128' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p3d-p-tp8-dp8-d-tp8-b32-c128-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p3d-p-tp8-dp8-d-tp8-b32-c128-stp.yaml new file mode 100644 index 0000000000..32d3bdef06 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p3d-p-tp8-dp8-d-tp8-b32-c128-stp.yaml @@ -0,0 +1,143 @@ +schema: 2 +name: b200-fp8-stp-low-latency-tp8-1p-3d + +dynamo: + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + source: + pypi: 0.9.1 + +model: + path: dsr1-fp8 + container: dynamo-sglang + precision: fp8 + +resources: + gpu_type: b200 + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + load-balance-method: round_robin + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 8192 + chunked-prefill-size: 65536 + max-running-requests: 8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + + decode: + nodes: 3 + workers: 3 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + context-length: 9600 + max-running-requests: 32 + cuda-graph-max-bs: 32 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + # disable-chunked-prefix-cache: true + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + concurrencies: '128' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p4d-p-tp8-dp8-d-tp8-b32-c128-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p4d-p-tp8-dp8-d-tp8-b32-c128-mtp.yaml new file mode 100644 index 0000000000..d8cda49d82 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p4d-p-tp8-dp8-d-tp8-b32-c128-mtp.yaml @@ -0,0 +1,145 @@ +schema: 2 +name: b200-fp8-mtp-low-latency-tep8-1p-4d + +dynamo: + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + source: + pypi: 0.9.1 + +model: + path: dsr1-fp8 + container: dynamo-sglang + precision: fp8 + +resources: + gpu_type: b200 + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + SGLANG_ENABLE_SPEC_V2: '1' + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + load-balance-method: round_robin + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 8192 + chunked-prefill-size: 65536 + max-running-requests: 8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + + decode: + nodes: 4 + workers: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + SGLANG_ENABLE_SPEC_V2: '1' + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + context-length: 9600 + max-running-requests: 32 + cuda-graph-max-bs: 32 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 720 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + concurrencies: '128' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p4d-p-tp8-dp8-d-tp8-b32-c128-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p4d-p-tp8-dp8-d-tp8-b32-c128-stp.yaml new file mode 100644 index 0000000000..b4f73851b9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p4d-p-tp8-dp8-d-tp8-b32-c128-stp.yaml @@ -0,0 +1,143 @@ +schema: 2 +name: b200-fp8-stp-low-latency-tp8-1p-4d + +dynamo: + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + source: + pypi: 0.9.1 + +model: + path: dsr1-fp8 + container: dynamo-sglang + precision: fp8 + +resources: + gpu_type: b200 + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + load-balance-method: round_robin + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 8192 + chunked-prefill-size: 65536 + max-running-requests: 8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + + decode: + nodes: 4 + workers: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + context-length: 9600 + max-running-requests: 32 + cuda-graph-max-bs: 32 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + # disable-chunked-prefix-cache: true + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + concurrencies: '128' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b22-c8x16x32x64x128-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b22-c8x16x32x64x128-mtp.yaml new file mode 100644 index 0000000000..320d9de2a5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b22-c8x16x32x64x128-mtp.yaml @@ -0,0 +1,145 @@ +schema: 2 +name: b200-fp8-mtp-low-latency-tep8-1p-6d + +dynamo: + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + source: + pypi: 0.9.1 + +model: + path: dsr1-fp8 + container: dynamo-sglang + precision: fp8 + +resources: + gpu_type: b200 + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + SGLANG_ENABLE_SPEC_V2: '1' + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + load-balance-method: round_robin + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 8192 + chunked-prefill-size: 65536 + max-running-requests: 8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + + decode: + nodes: 6 + workers: 6 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + SGLANG_ENABLE_SPEC_V2: '1' + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + context-length: 9600 + max-running-requests: 22 + cuda-graph-max-bs: 22 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 720 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + concurrencies: 8x16x32x64x128 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b22-c8x16x32x64x128-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b22-c8x16x32x64x128-stp.yaml new file mode 100644 index 0000000000..c045d20de4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b22-c8x16x32x64x128-stp.yaml @@ -0,0 +1,143 @@ +schema: 2 +name: b200-fp8-stp-low-latency-tp8-1p-6d + +dynamo: + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + source: + pypi: 0.9.1 + +model: + path: dsr1-fp8 + container: dynamo-sglang + precision: fp8 + +resources: + gpu_type: b200 + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + load-balance-method: round_robin + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 8192 + chunked-prefill-size: 65536 + max-running-requests: 8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + + decode: + nodes: 6 + workers: 6 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + context-length: 9600 + max-running-requests: 22 + cuda-graph-max-bs: 22 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + # disable-chunked-prefix-cache: true + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + concurrencies: 8x16x32x64x128 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-2p1d-p-tp8-dp8-d-tp8-ep8-dp8-b512-c512-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-2p1d-p-tp8-dp8-d-tp8-ep8-dp8-b512-c512-mtp.yaml new file mode 100644 index 0000000000..7c96ef3bce --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-2p1d-p-tp8-dp8-d-tp8-ep8-dp8-b512-c512-mtp.yaml @@ -0,0 +1,148 @@ +schema: 2 +name: b200-fp8-mtp-max-tpt-dep8-2p-1d + +dynamo: + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + source: + pypi: 0.9.1 + +model: + path: dsr1-fp8 + container: dynamo-sglang + precision: fp8 + +resources: + gpu_type: b200 + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 2 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + SGLANG_ENABLE_SPEC_V2: '1' + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + load-balance-method: round_robin + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 8192 + chunked-prefill-size: 65536 + max-running-requests: 8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + + decode: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + SGLANG_ENABLE_SPEC_V2: '1' + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + context-length: 9600 + max-running-requests: 512 + cuda-graph-max-bs: 64 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 720 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + concurrencies: '512' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-2p1d-p-tp8-dp8-d-tp8-ep8-dp8-b512-c512-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-2p1d-p-tp8-dp8-d-tp8-ep8-dp8-b512-c512-stp.yaml new file mode 100644 index 0000000000..b26a35931f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-2p1d-p-tp8-dp8-d-tp8-ep8-dp8-b512-c512-stp.yaml @@ -0,0 +1,144 @@ +schema: 2 +name: b200-fp8-stp-max-tpt-dep8-2p-1d + +dynamo: + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + source: + pypi: 0.9.1 + +model: + path: dsr1-fp8 + container: dynamo-sglang + precision: fp8 + +resources: + gpu_type: b200 + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 2 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + load-balance-method: round_robin + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 8192 + chunked-prefill-size: 65536 + max-running-requests: 8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + + decode: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + context-length: 9600 + max-running-requests: 512 + cuda-graph-max-bs: 512 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + concurrencies: '512' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-3p1d-p-tp8-dp8-d-tp8-ep8-dp8-b1024-c1024-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-3p1d-p-tp8-dp8-d-tp8-ep8-dp8-b1024-c1024-mtp.yaml new file mode 100644 index 0000000000..d1d1a67f5d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-3p1d-p-tp8-dp8-d-tp8-ep8-dp8-b1024-c1024-mtp.yaml @@ -0,0 +1,148 @@ +schema: 2 +name: b200-fp8-mtp-max-tpt-dep8-3p-1d + +dynamo: + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + source: + pypi: 0.9.1 + +model: + path: dsr1-fp8 + container: dynamo-sglang + precision: fp8 + +resources: + gpu_type: b200 + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 3 + workers: 3 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + SGLANG_ENABLE_SPEC_V2: '1' + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + load-balance-method: round_robin + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 8192 + chunked-prefill-size: 65536 + max-running-requests: 8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + + decode: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + SGLANG_ENABLE_SPEC_V2: '1' + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + context-length: 9600 + max-running-requests: 1024 + cuda-graph-max-bs: 128 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 720 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + concurrencies: '1024' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-3p1d-p-tp8-dp8-d-tp8-ep8-dp8-b1024-c1024-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-3p1d-p-tp8-dp8-d-tp8-ep8-dp8-b1024-c1024-stp.yaml new file mode 100644 index 0000000000..8f6cc2201c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/b200-fp8/8k1k/disagg-3p1d-p-tp8-dp8-d-tp8-ep8-dp8-b1024-c1024-stp.yaml @@ -0,0 +1,144 @@ +schema: 2 +name: b200-fp8-stp-max-tpt-dep8-3p-1d + +dynamo: + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + source: + pypi: 0.9.1 + +model: + path: dsr1-fp8 + container: dynamo-sglang + precision: fp8 + +resources: + gpu_type: b200 + gpus_per_node: 8 + +engine: sglang +roles: + prefill: + nodes: 3 + workers: 3 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + load-balance-method: round_robin + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 8192 + chunked-prefill-size: 65536 + max-running-requests: 8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + + decode: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + + args: + # Model configuration + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: fp8 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + context-length: 9600 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + # Attention + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + + # MoE + moe-runner-backend: flashinfer_trtllm + + # Other flags + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + concurrencies: '1024' diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp4/8k1k/disagg-10p1d-p-tp4-d-tp32-ep32-dp32-b16384-c2048-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp4/8k1k/disagg-10p1d-p-tp4-d-tp32-ep32-dp32-b16384-c2048-stp.yaml new file mode 100644 index 0000000000..6d5f76a41e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp4/8k1k/disagg-10p1d-p-tp4-d-tp32-ep32-dp32-b16384-c2048-stp.yaml @@ -0,0 +1,183 @@ +schema: 2 +name: "gb200-fp4-8k1k-max-tpt" + +dynamo: + request_plane: "nats" + + source: + pypi: 0.8.1 + # Use NATS for a recipe prior to Dynamo commit 39d2a68. +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 + nginx_container: nginx-sqsh + +model: + path: "dsr1-fp4" + container: "dynamo-sglang" + precision: "fp4" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: sglang +roles: + prefill: + nodes: 10 + workers: 10 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN: "1" + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + + # KV cache and attention + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + + # Quantization + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + + # Radix cache disabled + disable-radix-cache: true + disable-chunked-prefix-cache: true + + # Other flags + stream-interval: 50 + decode-log-interval: 1000 + watchdog-timeout: 1000000 + context-length: 9600 + disable-shared-experts-fusion: true + disaggregation-bootstrap-port: 30001 + + # Prefill-specific mode + disaggregation-mode: "prefill" + + # Memory and token limits + mem-fraction-static: 0.95 + max-total-tokens: 131072 + max-prefill-tokens: 524288 + chunked-prefill-size: 131072 + + # Request handling + max-running-requests: 30000 + load-balance-method: "round_robin" + + # Performance optimizations + disable-cuda-graph: true + enable-dp-attention: false + fp4-gemm-backend: "flashinfer_cutlass" + disaggregation-transfer-backend: nixl + + # Parallelism + tp-size: 4 + dp-size: 1 + ep-size: 1 + + decode: + nodes: 8 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN: "1" + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + + # KV cache and attention + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + + # Quantization + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_cutedsl" + + # Radix cache disabled + disable-radix-cache: true + disable-chunked-prefix-cache: true + + # Other flags + stream-interval: 50 + decode-log-interval: 1000 + watchdog-timeout: 1000000 + context-length: 9600 + disable-shared-experts-fusion: true + eplb-algorithm: "deepseek" + disaggregation-bootstrap-port: 30001 + + # Decode-specific mode + disaggregation-mode: "decode" + + # Memory and token limits + mem-fraction-static: 0.83 + max-total-tokens: 524288 + chunked-prefill-size: 24576 + + # Request handling + max-running-requests: 16384 + + # DeepEP configuration + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + ep-num-redundant-experts: 32 + + cuda-graph-max-bs: 512 + num-reserved-decode-tokens: 112 + + # Additional decode optimizations + moe-dense-tp-size: 1 + enable-dp-lm-head: true + prefill-round-robin-balance: true + enable-dp-attention: true + fp4-gemm-backend: "flashinfer_cutlass" + disaggregation-transfer-backend: nixl + + # Parallelism + tp-size: 32 + dp-size: 32 + ep-size: 32 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: 700 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp4/8k1k/disagg-1p4d-p-tp4-d-tp4-c4x8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp4/8k1k/disagg-1p4d-p-tp4-d-tp4-c4x8-stp.yaml new file mode 100644 index 0000000000..0793b450a9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp4/8k1k/disagg-1p4d-p-tp4-d-tp4-c4x8-stp.yaml @@ -0,0 +1,123 @@ +schema: 2 +name: "gb200-fp4-8k1k-low-latency" + +dynamo: + request_plane: "nats" + + source: + pypi: 0.8.1 + # Use NATS for a recipe prior to Dynamo commit 39d2a68. +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_container: nginx-sqsh + +model: + path: "dsr1-fp4" + container: "dynamo-sglang" + precision: "fp4" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + + args: + disaggregation-mode: "prefill" + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.95 + max-total-tokens: 32768 + chunked-prefill-size: 24576 + cuda-graph-max-bs: 256 + max-running-requests: 512 + scheduler-recv-interval: 10 + enable-symm-mem: true + load-balance-method: "round_robin" + disaggregation-bootstrap-port: 30001 + data-parallel-size: 1 + disaggregation-transfer-backend: nixl + fp4-gemm-backend: "flashinfer_trtllm" + tensor-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: false + + decode: + nodes: 4 + workers: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + + args: + disaggregation-mode: "decode" + served-model-name: "deepseek-ai/DeepSeek-R1" + prefill-round-robin-balance: true + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + disaggregation-bootstrap-port: 30001 + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.95 + chunked-prefill-size: 8192 + cuda-graph-max-bs: 256 + scheduler-recv-interval: 10 + enable-symm-mem: true + disaggregation-transfer-backend: nixl + fp4-gemm-backend: "flashinfer_trtllm" + tensor-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: false + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4x8" + req_rate: 300 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp4/8k1k/disagg-6p1d-p-tp4-d-tp48-ep48-dp48-b16384-c512x2048x4096-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp4/8k1k/disagg-6p1d-p-tp4-d-tp48-ep48-dp48-b16384-c512x2048x4096-stp.yaml new file mode 100644 index 0000000000..db1ef476f2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp4/8k1k/disagg-6p1d-p-tp4-d-tp48-ep48-dp48-b16384-c512x2048x4096-stp.yaml @@ -0,0 +1,183 @@ +schema: 2 +name: "gb200-fp4-8k1k-mid-curve" + +dynamo: + request_plane: "nats" + + source: + pypi: 0.8.1 + # Use NATS for a recipe prior to Dynamo commit 39d2a68. +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 + nginx_container: nginx-sqsh + +model: + path: "dsr1-fp4" + container: "dynamo-sglang" + precision: "fp4" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: sglang +roles: + prefill: + nodes: 6 + workers: 6 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN: "1" + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + + # KV cache and attention + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + + # Quantization + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + + # Radix cache disabled + disable-radix-cache: true + disable-chunked-prefix-cache: true + + # Other flags + stream-interval: 50 + decode-log-interval: 1000 + watchdog-timeout: 1000000 + context-length: 9600 + disable-shared-experts-fusion: true + disaggregation-bootstrap-port: 30001 + + # Prefill-specific mode + disaggregation-mode: "prefill" + + # Memory and token limits + mem-fraction-static: 0.95 + max-total-tokens: 131072 + max-prefill-tokens: 524288 + chunked-prefill-size: 131072 + + # Request handling + max-running-requests: 30000 + load-balance-method: "round_robin" + + # Performance optimizations + disable-cuda-graph: true + enable-dp-attention: false + fp4-gemm-backend: "flashinfer_cutlass" + disaggregation-transfer-backend: nixl + + # Parallelism + tp-size: 4 + dp-size: 1 + ep-size: 1 + + decode: + nodes: 12 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN: "1" + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + + # KV cache and attention + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + + # Quantization + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_cutedsl" + + # Radix cache disabled + disable-radix-cache: true + disable-chunked-prefix-cache: true + + # Other flags + stream-interval: 50 + decode-log-interval: 1000 + watchdog-timeout: 1000000 + context-length: 9600 + disable-shared-experts-fusion: true + eplb-algorithm: "deepseek" + disaggregation-bootstrap-port: 30001 + + # Decode-specific mode + disaggregation-mode: "decode" + + # Memory and token limits + mem-fraction-static: 0.83 + max-total-tokens: 524288 + chunked-prefill-size: 24576 + + # Request handling + max-running-requests: 16384 + + # DeepEP configuration + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + ep-num-redundant-experts: 32 + + cuda-graph-max-bs: 512 + num-reserved-decode-tokens: 112 + + # Additional decode optimizations + moe-dense-tp-size: 1 + enable-dp-lm-head: true + prefill-round-robin-balance: true + enable-dp-attention: true + fp4-gemm-backend: "flashinfer_cutlass" + disaggregation-transfer-backend: nixl + + # Parallelism + tp-size: 48 + dp-size: 48 + ep-size: 48 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512x2048x4096" + req_rate: 700 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp8/8k1k/disagg-5p1d-p-tp8-ep8-dp8-d-tp32-ep32-dp32-b8192-c512x1024x2048x6144-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp8/8k1k/disagg-5p1d-p-tp8-ep8-dp8-d-tp32-ep32-dp32-b8192-c512x1024x2048x6144-stp.yaml new file mode 100644 index 0000000000..344a5b3391 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp8/8k1k/disagg-5p1d-p-tp8-ep8-dp8-d-tp32-ep32-dp32-b8192-c512x1024x2048x6144-stp.yaml @@ -0,0 +1,174 @@ +schema: 2 +name: "gb200-8k1k-fp8-mid-tpt" + +dynamo: + request_plane: "nats" + + source: + pypi: 0.8.1 + # Use NATS for a recipe prior to Dynamo commit 39d2a68. +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 + nginx_container: nginx + +model: + path: "dsr1-fp8" + container: "dynamo-sglang" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: sglang +roles: + prefill: + nodes: 10 + workers: 5 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + MC_TE_METRIC: "true" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + PYTHONUNBUFFERED: "1" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + skip-tokenizer-init: true + trust-remote-code: true + + # Parallelism + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + + # KV cache and attention + attention-backend: "trtllm_mla" + kv-cache-dtype: "fp8_e4m3" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + stream-interval: 50 + max-running-requests: 30000 + context-length: 9300 + watchdog-timeout: 1000000 + disable-shared-experts-fusion: true + eplb-algorithm: "deepseek" + + # Prefill-specific mode + disaggregation-mode: "prefill" + + # Memory and token limits + mem-fraction-static: 0.80 + max-total-tokens: 524288 + chunked-prefill-size: 131072 + + # Request handling + load-balance-method: "round_robin" + + # Performance optimizations + disable-cuda-graph: true + + # DeepEP configuration + moe-a2a-backend: "deepep" + deepep-mode: "normal" + ep-dispatch-algorithm: "dynamic" + moe-dense-tp-size: 1 + enable-dp-lm-head: true + ep-num-redundant-experts: 32 + deepep-config: "/configs/deepep_config.json" + disaggregation-bootstrap-port: 30001 + disaggregation-transfer-backend: nixl + + decode: + nodes: 8 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "256" + MC_TE_METRIC: "true" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + PYTHONUNBUFFERED: "1" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + skip-tokenizer-init: true + trust-remote-code: true + + # Parallelism + tp-size: 32 + dp-size: 32 + ep-size: 32 + enable-dp-attention: true + + # KV cache and attention + attention-backend: "trtllm_mla" + kv-cache-dtype: "fp8_e4m3" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + stream-interval: 50 + decode-log-interval: 1000 + max-running-requests: 8192 + context-length: 9300 + watchdog-timeout: 1000000 + disable-shared-experts-fusion: true + eplb-algorithm: "deepseek" + + # Decode-specific mode + disaggregation-mode: "decode" + + # Memory and token limits + mem-fraction-static: 0.82 + chunked-prefill-size: 36864 + + # DeepEP configuration + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + moe-dense-tp-size: 1 + enable-dp-lm-head: true + prefill-round-robin-balance: true + ep-num-redundant-experts: 32 + deepep-config: "/configs/deepep_config.json" + # CUDA graphs + cuda-graph-max-bs: 256 + disaggregation-bootstrap-port: 30001 + disaggregation-transfer-backend: nixl + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512x1024x2048x6144" + req_rate: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp8/8k1k/disagg-6p1d-p-tp8-ep8-dp8-d-tp24-ep24-dp24-b8192-c2048x4096x6144-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp8/8k1k/disagg-6p1d-p-tp8-ep8-dp8-d-tp24-ep24-dp24-b8192-c2048x4096x6144-stp.yaml new file mode 100644 index 0000000000..6a22f63b68 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp8/8k1k/disagg-6p1d-p-tp8-ep8-dp8-d-tp24-ep24-dp24-b8192-c2048x4096x6144-stp.yaml @@ -0,0 +1,177 @@ +schema: 2 +name: "gb200-8k1k-fp8-max-tpt" + +dynamo: + request_plane: "nats" + + source: + pypi: 0.8.1 + # Use NATS for a recipe prior to Dynamo commit 39d2a68. +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 + nginx_container: nginx + +model: + path: "dsr1-fp8" + container: "dynamo-sglang" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: sglang +roles: + prefill: + nodes: 12 + workers: 6 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + MC_TE_METRIC: "true" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + PYTHONUNBUFFERED: "1" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + skip-tokenizer-init: true + trust-remote-code: true + + # Parallelism + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + + # KV cache and attention + attention-backend: "trtllm_mla" + kv-cache-dtype: "fp8_e4m3" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + stream-interval: 50 + max-running-requests: 30000 + context-length: 9300 + watchdog-timeout: 1000000 + disable-shared-experts-fusion: true + eplb-algorithm: "deepseek" + + # Prefill-specific mode + disaggregation-mode: "prefill" + + # Memory and token limits + mem-fraction-static: 0.80 + max-total-tokens: 524288 + chunked-prefill-size: 131072 + + # Request handling + load-balance-method: "round_robin" + + # Performance optimizations + disable-cuda-graph: true + + # DeepEP configuration + moe-a2a-backend: "deepep" + deepep-mode: "normal" + ep-dispatch-algorithm: "dynamic" + moe-dense-tp-size: 1 + enable-dp-lm-head: true + ep-num-redundant-experts: 32 + deepep-config: "/configs/deepep_config.json" + disaggregation-bootstrap-port: 30001 + disaggregation-transfer-backend: nixl + + decode: + nodes: 6 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + MC_TE_METRIC: "true" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + PYTHONUNBUFFERED: "1" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + skip-tokenizer-init: true + trust-remote-code: true + + # Parallelism + tp-size: 24 + dp-size: 24 + ep-size: 24 + enable-dp-attention: true + + # KV cache and attention + attention-backend: "trtllm_mla" + kv-cache-dtype: "fp8_e4m3" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + stream-interval: 50 + decode-log-interval: 1000 + max-running-requests: 8192 + context-length: 9300 + watchdog-timeout: 1000000 + disable-shared-experts-fusion: true + eplb-algorithm: "deepseek" + + # Decode-specific mode + disaggregation-mode: "decode" + + # Memory and token limits + mem-fraction-static: 0.82 + chunked-prefill-size: 36864 + + # DeepEP configuration + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + moe-dense-tp-size: 1 + enable-dp-lm-head: true + prefill-round-robin-balance: true + ep-num-redundant-experts: 32 + deepep-config: "/configs/deepep_config.json" + # CUDA graphs + cuda-graph-bs: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, + 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 264, 272, 280, 288, 296, 304, 312, 320, 328, 336, + 344, 352, 360, 368, 376, 384, 416, 448, 480, 512] + cuda-graph-max-bs: 512 + disaggregation-bootstrap-port: 30001 + disaggregation-transfer-backend: nixl + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048x4096x6144" + req_rate: "300" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp8/8k1k/disagg-tp8-stp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp8/8k1k/disagg-tp8-stp-variants.yaml new file mode 100644 index 0000000000..2ab4bec729 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb200-fp8/8k1k/disagg-tp8-stp-variants.yaml @@ -0,0 +1,135 @@ +schema: 2 +base: + name: "gb200-fp8-8k1k-low-latency" + + dynamo: + request_plane: "nats" + + source: + pypi: 0.8.1 + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 2 + nginx_container: nginx + + model: + path: "dsr1-fp8" + container: "dynamo-sglang" + precision: "fp8" + + resources: + gpu_type: "gb200" + gpus_per_node: 4 + engine: sglang + roles: + prefill: + nodes: 2 + workers: 1 + + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "fp8" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + watchdog-timeout: 1000000 + context-length: 9600 + disaggregation-mode: "prefill" + mem-fraction-static: 0.6 + max-running-requests: 32 + cuda-graph-max-bs: 32 + load-balance-method: "round_robin" + scheduler-recv-interval: 10 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + fp8-gemm-backend: "flashinfer_trtllm" + disaggregation-bootstrap-port: 30001 + disaggregation-transfer-backend: nixl + + decode: + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "fp8" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + watchdog-timeout: 1000000 + context-length: 9600 + disaggregation-mode: "decode" + mem-fraction-static: 0.7 + cuda-graph-max-bs: 32 + max-running-requests: 32 + scheduler-recv-interval: 10 + enable-symm-mem: true + prefill-round-robin-balance: true + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + fp8-gemm-backend: "flashinfer_trtllm" + disaggregation-bootstrap-port: 30001 + disaggregation-transfer-backend: nixl + + benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4x8x16" + req_rate: "inf" + +zip_override_lowlat: + resources: {} + name: + - "gb200-fp8-8k1k-low-latency-c8" + - "gb200-fp8-8k1k-low-latency-c16-32" + - "gb200-fp8-8k1k-low-latency-c64" + roles: + prefill: + args: + max-prefill-tokens: [8192, 16384, 32768] + chunked-prefill-size: [8192, 16384, 32768] + decode: + nodes: [14, 14, 6] + workers: [7, 7, 3] + args: + max-running-requests: [1, 4, 16] + benchmark: + concurrencies: ["8", "16x32", "64"] diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp4/8k1k/disagg-10p1d-p-tp4-d-tp32-ep32-dp32-b16384-c2048-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp4/8k1k/disagg-10p1d-p-tp4-d-tp32-ep32-dp32-b16384-c2048-stp.yaml new file mode 100644 index 0000000000..f7b29609f5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp4/8k1k/disagg-10p1d-p-tp4-d-tp32-ep32-dp32-b16384-c2048-stp.yaml @@ -0,0 +1,182 @@ +schema: 2 +name: "gb300-fp4-8k1k-max-tpt" + +dynamo: + request_plane: "nats" + source: + pypi: 0.8.1 + # Use NATS for a recipe prior to Dynamo commit 39d2a68. +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 + nginx_container: nginx-sqsh + +model: + path: "dsr1" + container: "dynamo-sglang" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: sglang +roles: + prefill: + nodes: 10 + workers: 10 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN: "1" + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + + # KV cache and attention + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + + # Quantization + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + + # Radix cache disabled + disable-radix-cache: true + disable-chunked-prefix-cache: true + + # Other flags + stream-interval: 50 + decode-log-interval: 1000 + watchdog-timeout: 1000000 + context-length: 9600 + disable-shared-experts-fusion: true + disaggregation-bootstrap-port: 30001 + + # Prefill-specific mode + disaggregation-mode: "prefill" + + # Memory and token limits + mem-fraction-static: 0.95 + max-total-tokens: 131072 + max-prefill-tokens: 524288 + chunked-prefill-size: 131072 + + # Request handling + max-running-requests: 30000 + load-balance-method: "round_robin" + + # Performance optimizations + disable-cuda-graph: true + enable-dp-attention: false + fp4-gemm-backend: "flashinfer_cutlass" + disaggregation-transfer-backend: nixl + + # Parallelism + tp-size: 4 + dp-size: 1 + ep-size: 1 + + decode: + nodes: 8 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN: "1" + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + + # KV cache and attention + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + + # Quantization + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_cutedsl" + + # Radix cache disabled + disable-radix-cache: true + disable-chunked-prefix-cache: true + + # Other flags + stream-interval: 50 + decode-log-interval: 1000 + watchdog-timeout: 1000000 + context-length: 9600 + disable-shared-experts-fusion: true + eplb-algorithm: "deepseek" + disaggregation-bootstrap-port: 30001 + + # Decode-specific mode + disaggregation-mode: "decode" + + # Memory and token limits + mem-fraction-static: 0.83 + max-total-tokens: 524288 + chunked-prefill-size: 24576 + + # Request handling + max-running-requests: 16384 + + # DeepEP configuration + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + ep-num-redundant-experts: 32 + + cuda-graph-max-bs: 512 + num-reserved-decode-tokens: 112 + + # Additional decode optimizations + moe-dense-tp-size: 1 + enable-dp-lm-head: true + prefill-round-robin-balance: true + enable-dp-attention: true + fp4-gemm-backend: "flashinfer_cutlass" + disaggregation-transfer-backend: nixl + + # Parallelism + tp-size: 32 + dp-size: 32 + ep-size: 32 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: 700 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp4/8k1k/disagg-1p4d-p-tp4-d-tp4-c4x8x32x64-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp4/8k1k/disagg-1p4d-p-tp4-d-tp4-c4x8x32x64-stp.yaml new file mode 100644 index 0000000000..d150e71670 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp4/8k1k/disagg-1p4d-p-tp4-d-tp4-c4x8x32x64-stp.yaml @@ -0,0 +1,124 @@ +schema: 2 +name: "gb300-8k1k-fp4-low-latency-8k1k" + +dynamo: + request_plane: "nats" + + source: + pypi: 0.8.1 + # Use NATS for a recipe prior to Dynamo commit 39d2a68. +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 3 + nginx_container: nginx-sqsh + +model: + path: "dsr1" + container: "dynamo-sglang" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + + args: + disaggregation-mode: "prefill" + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.95 + max-total-tokens: 32768 + chunked-prefill-size: 24576 + cuda-graph-max-bs: 256 + max-running-requests: 512 + scheduler-recv-interval: 10 + enable-symm-mem: true + load-balance-method: "round_robin" + disaggregation-bootstrap-port: 30001 + data-parallel-size: 1 + tensor-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: false + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + + + decode: + nodes: 4 + workers: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + + args: + disaggregation-mode: "decode" + served-model-name: "deepseek-ai/DeepSeek-R1" + prefill-round-robin-balance: true + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + disaggregation-bootstrap-port: 30001 + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.95 + chunked-prefill-size: 8192 + cuda-graph-max-bs: 128 + scheduler-recv-interval: 10 + enable-symm-mem: true + tensor-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: false + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4x8x32x64" + req_rate: 300 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp4/8k1k/disagg-6p1d-p-tp4-d-tp48-ep48-dp48-b16384-c512x2048x4096-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp4/8k1k/disagg-6p1d-p-tp4-d-tp48-ep48-dp48-b16384-c512x2048x4096-stp.yaml new file mode 100644 index 0000000000..e83bebe1cc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp4/8k1k/disagg-6p1d-p-tp4-d-tp48-ep48-dp48-b16384-c512x2048x4096-stp.yaml @@ -0,0 +1,183 @@ +schema: 2 +name: "gb300-fp4-8k1k-mid-curve" + +dynamo: + request_plane: "nats" + + source: + pypi: 0.8.1 + # Use NATS for a recipe prior to Dynamo commit 39d2a68. +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 + nginx_container: nginx-sqsh + +model: + path: "dsr1" + container: "dynamo-sglang" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: sglang +roles: + prefill: + nodes: 6 + workers: 6 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN: "1" + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + + # KV cache and attention + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + + # Quantization + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + + # Radix cache disabled + disable-radix-cache: true + disable-chunked-prefix-cache: true + + # Other flags + stream-interval: 50 + decode-log-interval: 1000 + watchdog-timeout: 1000000 + context-length: 9600 + disable-shared-experts-fusion: true + disaggregation-bootstrap-port: 30001 + + # Prefill-specific mode + disaggregation-mode: "prefill" + + # Memory and token limits + mem-fraction-static: 0.95 + max-total-tokens: 131072 + max-prefill-tokens: 524288 + chunked-prefill-size: 131072 + + # Request handling + max-running-requests: 30000 + load-balance-method: "round_robin" + + # Performance optimizations + disable-cuda-graph: true + enable-dp-attention: false + fp4-gemm-backend: "flashinfer_cutlass" + disaggregation-transfer-backend: nixl + + # Parallelism + tp-size: 4 + dp-size: 1 + ep-size: 1 + + decode: + nodes: 12 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN: "1" + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + + # KV cache and attention + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + + # Quantization + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_cutedsl" + + # Radix cache disabled + disable-radix-cache: true + disable-chunked-prefix-cache: true + + # Other flags + stream-interval: 50 + decode-log-interval: 1000 + watchdog-timeout: 1000000 + context-length: 9600 + disable-shared-experts-fusion: true + eplb-algorithm: "deepseek" + disaggregation-bootstrap-port: 30001 + + # Decode-specific mode + disaggregation-mode: "decode" + + # Memory and token limits + mem-fraction-static: 0.83 + max-total-tokens: 524288 + chunked-prefill-size: 24576 + + # Request handling + max-running-requests: 16384 + + # DeepEP configuration + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + ep-num-redundant-experts: 32 + + cuda-graph-max-bs: 512 + num-reserved-decode-tokens: 112 + + # Additional decode optimizations + moe-dense-tp-size: 1 + enable-dp-lm-head: true + prefill-round-robin-balance: true + enable-dp-attention: true + fp4-gemm-backend: "flashinfer_cutlass" + disaggregation-transfer-backend: nixl + + # Parallelism + tp-size: 48 + dp-size: 48 + ep-size: 48 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512x2048x4096" + req_rate: 700 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c4x8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c4x8-stp.yaml new file mode 100644 index 0000000000..e339b0137a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c4x8-stp.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: "gb300-8k1k-fp8-low-latency" + +model: + path: "dsr1-fp8" + container: "dynamo-sglang" + precision: "fp8" + +frontend: + nginx_container: nginx + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +slurm: + time_limit: "02:00:00" + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DG_CACHE_DIR: "/configs/dg-10212025" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + # SGLANG_ENABLE_FLASHINFER_GEMM: "1" # deprecated in 0.5.7, --fp8-gemm-backend=flashinfer_trtllm + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "fp8" + moe-runner-backend: "flashinfer_trtllm" + fp8-gemm-backend: "flashinfer_trtllm" + disable-radix-cache: true + stream-interval: 10 + watchdog-timeout: 1000000 + context-length: 9300 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.95 + max-total-tokens: 32768 + chunked-prefill-size: 32768 + max-prefill-tokens: 32768 + cuda-graph-max-bs: 128 + max-running-requests: 128 + load-balance-method: "round_robin" + scheduler-recv-interval: 10 + enable-flashinfer-allreduce-fusion: false # to save mem + enable-symm-mem: false # to save mem + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + decode: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DG_CACHE_DIR: "/configs/dg-10212025" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + # SGLANG_ENABLE_FLASHINFER_GEMM: "1" # deprecated in 0.5.7, --fp8-gemm-backend=flashinfer_trtllm + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "fp8" + moe-runner-backend: "flashinfer_trtllm" + fp8-gemm-backend: "flashinfer_trtllm" + disable-radix-cache: true + stream-interval: 10 + watchdog-timeout: 1000000 + context-length: 9300 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + chunked-prefill-size: -1 # save mem + cuda-graph-max-bs: 128 + max-running-requests: 128 + scheduler-recv-interval: 1 # save mem + enable-flashinfer-allreduce-fusion: false # to save mem + enable-symm-mem: false # to save mem + prefill-round-robin-balance: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [4, 8] + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp8/8k1k/disagg-5p1d-p-tp8-ep8-dp8-d-tp32-ep32-dp32-b45000-c128x256x512x1024-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp8/8k1k/disagg-5p1d-p-tp8-ep8-dp8-d-tp32-ep32-dp32-b45000-c128x256x512x1024-stp.yaml new file mode 100644 index 0000000000..88f207b6e7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp8/8k1k/disagg-5p1d-p-tp8-ep8-dp8-d-tp32-ep32-dp32-b45000-c128x256x512x1024-stp.yaml @@ -0,0 +1,180 @@ +# GB300 FP8 Mid Throughput Configuration + +schema: 2 +name: "gb300-8k1k-fp8-mid" + +model: + path: "dsr1-fp8" + container: "dynamo-sglang" + precision: "fp8" + +frontend: + nginx_container: nginx + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +engine: sglang +roles: + prefill: + nodes: 10 + workers: 5 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_DG_CACHE_DIR: "/configs/dg-10212025" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + MC_TE_METRIC: "true" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + PYTHONUNBUFFERED: "1" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + skip-tokenizer-init: true + trust-remote-code: true + + # Parallelism + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + + # KV cache and attention + attention-backend: "trtllm_mla" + kv-cache-dtype: "fp8_e4m3" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + stream-interval: 50 + max-running-requests: 30000 + context-length: 9300 + watchdog-timeout: 1000000 + disable-shared-experts-fusion: true + eplb-algorithm: "deepseek" + disaggregation-bootstrap-port: 30001 + disaggregation-transfer-backend: nixl + + # Prefill-specific mode + disaggregation-mode: "prefill" + + # Memory and token limits + mem-fraction-static: 0.75 + max-total-tokens: 524288 + chunked-prefill-size: 131072 + + # Request handling + load-balance-method: "round_robin" + + # Performance optimizations + disable-cuda-graph: true + + # DeepEP configuration + moe-a2a-backend: "deepep" + deepep-mode: "normal" + ep-dispatch-algorithm: "dynamic" + moe-dense-tp-size: 1 + enable-dp-lm-head: true + ep-num-redundant-experts: 32 + deepep-config: "/configs/deepep_config.json" + + decode: + nodes: 8 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_DG_CACHE_DIR: "/configs/dg-10212025" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "768" + MC_TE_METRIC: "true" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + PYTHONUNBUFFERED: "1" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + skip-tokenizer-init: true + trust-remote-code: true + disaggregation-transfer-backend: nixl + + # Parallelism + tp-size: 32 + dp-size: 32 + ep-size: 32 + enable-dp-attention: true + + # KV cache and attention + attention-backend: "trtllm_mla" + kv-cache-dtype: "fp8_e4m3" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + stream-interval: 50 + decode-log-interval: 1000 + max-running-requests: 45000 + context-length: 9300 + + watchdog-timeout: 1000000 + disable-shared-experts-fusion: true + eplb-algorithm: "deepseek" + disaggregation-bootstrap-port: 30001 + + # Decode-specific mode + disaggregation-mode: "decode" + + # Memory and token limits + mem-fraction-static: 0.82 + chunked-prefill-size: 36864 + + # DeepEP configuration + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + moe-dense-tp-size: 1 + enable-dp-lm-head: true + prefill-round-robin-balance: true + ep-num-redundant-experts: 32 + deepep-config: "/configs/deepep_config.json" + + # CUDA graphs + cuda-graph-bs: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, + 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 264, 272, 280, 288, 296, 304, 312, 320, 328, 336, + 344, 352, 360, 368, 376, 384, 416, 448, 480, 512, 544, 576, 608, 640, 672, 704, 736, 768] + cuda-graph-max-bs: 768 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [128, 256, 512, 1024] + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp8/8k1k/disagg-6p1d-p-tp8-ep8-dp8-d-tp24-ep24-dp24-b45000-c2048x4096-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp8/8k1k/disagg-6p1d-p-tp8-ep8-dp8-d-tp24-ep24-dp24-b45000-c2048x4096-stp.yaml new file mode 100644 index 0000000000..c8bf36e34a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/gb300-fp8/8k1k/disagg-6p1d-p-tp8-ep8-dp8-d-tp24-ep24-dp24-b45000-c2048x4096-stp.yaml @@ -0,0 +1,180 @@ +# GB300 FP8 Max Throughput Configuration + +schema: 2 +name: "gb300-8k1k-fp8-max" + +model: + path: "dsr1-fp8" + container: "dynamo-sglang" + precision: "fp8" + +frontend: + nginx_container: nginx + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +engine: sglang +roles: + prefill: + nodes: 12 + workers: 6 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_DG_CACHE_DIR: "/configs/dg-10212025" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + MC_TE_METRIC: "true" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + PYTHONUNBUFFERED: "1" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + skip-tokenizer-init: true + trust-remote-code: true + + # Parallelism + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + + # KV cache and attention + attention-backend: "trtllm_mla" + kv-cache-dtype: "fp8_e4m3" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + stream-interval: 50 + max-running-requests: 30000 + context-length: 9300 + watchdog-timeout: 1000000 + disable-shared-experts-fusion: true + eplb-algorithm: "deepseek" + disaggregation-bootstrap-port: 30001 + disaggregation-transfer-backend: nixl + + # Prefill-specific mode + disaggregation-mode: "prefill" + + # Memory and token limits + mem-fraction-static: 0.75 + max-total-tokens: 524288 + chunked-prefill-size: 131072 + + # Request handling + load-balance-method: "round_robin" + + # Performance optimizations + disable-cuda-graph: true + + # DeepEP configuration + moe-a2a-backend: "deepep" + deepep-mode: "normal" + ep-dispatch-algorithm: "dynamic" + moe-dense-tp-size: 1 + enable-dp-lm-head: true + ep-num-redundant-experts: 32 + deepep-config: "/configs/deepep_config.json" + + decode: + nodes: 6 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_DG_CACHE_DIR: "/configs/dg-10212025" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "768" + MC_TE_METRIC: "true" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + PYTHONUNBUFFERED: "1" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + skip-tokenizer-init: true + trust-remote-code: true + disaggregation-transfer-backend: nixl + + # Parallelism + tp-size: 24 + dp-size: 24 + ep-size: 24 + enable-dp-attention: true + + # KV cache and attention + attention-backend: "trtllm_mla" + kv-cache-dtype: "fp8_e4m3" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + stream-interval: 50 + decode-log-interval: 1000 + max-running-requests: 45000 + context-length: 9300 + + watchdog-timeout: 1000000 + disable-shared-experts-fusion: true + eplb-algorithm: "deepseek" + disaggregation-bootstrap-port: 30001 + + # Decode-specific mode + disaggregation-mode: "decode" + + # Memory and token limits + mem-fraction-static: 0.82 + chunked-prefill-size: 36864 + + # DeepEP configuration + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + moe-dense-tp-size: 1 + enable-dp-lm-head: true + prefill-round-robin-balance: true + ep-num-redundant-experts: 32 + deepep-config: "/configs/deepep_config.json" + + # CUDA graphs + cuda-graph-bs: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, + 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 264, 272, 280, 288, 296, 304, 312, 320, 328, 336, + 344, 352, 360, 368, 376, 384, 416, 448, 480, 512, 544, 576, 608, 640, 672, 704, 736, 768] + cuda-graph-max-bs: 768 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [2048, 4096] + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-b128-c1x2x4x8x16x32x64x128-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-b128-c1x2x4x8x16x32x64x128-mtp.yaml new file mode 100644 index 0000000000..f08550fd32 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-b128-c1x2x4x8x16x32x64x128-mtp.yaml @@ -0,0 +1,124 @@ +schema: 2 +name: "h100-fp8-1p1d-max-tp-mtp" + +model: + path: "dsr1-fp8" + container: "lmsysorg/sglang:v0.5.8-cu130" + precision: "fp8" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +frontend: + nginx_container: nginx-sqsh + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_SPEC_V2: "1" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 16 + dp-size: 1 + ep-size: 1 + enable-dp-attention: false + + # KV cache and attention + attention-backend: "flashinfer" + + # Radix cache disabled + disable-radix-cache: true + + # Prefill capacity + max-running-requests: 2 + + # Prefill-specific mode + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 2048 + chunked-prefill-size: 2048 + + # Request handling + load-balance-method: "round_robin" + + # MTP (Multi-Token Prediction) + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + + decode: + nodes: 2 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_SPEC_V2: "1" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 16 + dp-size: 1 + ep-size: 1 + enable-dp-attention: false + + # KV cache and attention + attention-backend: "flashinfer" + + # Other flags + disable-radix-cache: true + stream-interval: 1 + + # Disagg + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + + # MTP (Multi-Token Prediction) + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64x128" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-b128-c1x2x4x8x16x32x64x128-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-b128-c1x2x4x8x16x32x64x128-stp.yaml new file mode 100644 index 0000000000..e3507c45ca --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-b128-c1x2x4x8x16x32x64x128-stp.yaml @@ -0,0 +1,110 @@ +schema: 2 +name: "h100-fp8-1p1d-max-tp" + +model: + path: "dsr1-fp8" + container: "lmsysorg/sglang:v0.5.8-cu130" + precision: "fp8" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +frontend: + nginx_container: nginx-sqsh + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 16 + dp-size: 1 + ep-size: 1 + enable-dp-attention: false + + # KV cache and attention + attention-backend: "flashinfer" + + # Radix cache disabled + disable-radix-cache: true + + # Prefill capacity + max-running-requests: 2 + + # Prefill-specific mode + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 2048 + chunked-prefill-size: 2048 + + # Request handling + load-balance-method: "round_robin" + + decode: + nodes: 2 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 16 + dp-size: 1 + ep-size: 1 + enable-dp-attention: false + + # KV cache and attention + attention-backend: "flashinfer" + + # Other flags + disable-radix-cache: true + stream-interval: 1 + + # Disagg + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64x128" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-ep16-dp16-b64-c1x2x4x8x16x32x64-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-ep16-dp16-b64-c1x2x4x8x16x32x64-mtp.yaml new file mode 100644 index 0000000000..e6551f3905 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-ep16-dp16-b64-c1x2x4x8x16x32x64-mtp.yaml @@ -0,0 +1,124 @@ +schema: 2 +name: "h100-fp8-1p1d-max-dep-mtp" + +model: + path: "dsr1-fp8" + container: "lmsysorg/sglang:v0.5.8-cu130" + precision: "fp8" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +frontend: + nginx_container: nginx-sqsh + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_SPEC_V2: "1" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 16 + dp-size: 1 + ep-size: 1 + enable-dp-attention: false + + # KV cache and attention + attention-backend: "flashinfer" + + # Radix cache disabled + disable-radix-cache: true + + # Prefill capacity + max-running-requests: 4 + + # Prefill-specific mode + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 2048 + chunked-prefill-size: 2048 + + # Request handling + load-balance-method: "round_robin" + + # MTP (Multi-Token Prediction) + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + + decode: + nodes: 2 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_SPEC_V2: "1" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 16 + dp-size: 16 + ep-size: 16 + enable-dp-attention: true + + # KV cache and attention + attention-backend: "flashinfer" + + # Other flags + disable-radix-cache: true + stream-interval: 1 + + # Disagg + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.85 + max-running-requests: 64 + cuda-graph-max-bs: 64 + + # MTP + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-ep16-dp16-b64-c1x2x4x8x16x32x64-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-ep16-dp16-b64-c1x2x4x8x16x32x64-stp.yaml new file mode 100644 index 0000000000..2ece076697 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-ep16-dp16-b64-c1x2x4x8x16x32x64-stp.yaml @@ -0,0 +1,110 @@ +schema: 2 +name: "h100-fp8-1p1d-max-dep" + +model: + path: "dsr1-fp8" + container: "lmsysorg/sglang:v0.5.8-cu130" + precision: "fp8" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +frontend: + nginx_container: nginx-sqsh + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 16 + dp-size: 1 + ep-size: 1 + enable-dp-attention: false + + # KV cache and attention + attention-backend: "flashinfer" + + # Radix cache disabled + disable-radix-cache: true + + # Prefill capacity + max-running-requests: 4 + + # Prefill-specific mode + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.6 + max-prefill-tokens: 2048 + chunked-prefill-size: 2048 + + # Request handling + load-balance-method: "round_robin" + + decode: + nodes: 2 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 16 + dp-size: 16 + ep-size: 16 + enable-dp-attention: true + + # KV cache and attention + attention-backend: "flashinfer" + + # Other flags + disable-radix-cache: true + stream-interval: 1 + + # Disagg + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.9 + max-running-requests: 64 + cuda-graph-max-bs: 64 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs128-1p1d-dep-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs128-1p1d-dep-mtp.yaml new file mode 100644 index 0000000000..50c761b46b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs128-1p1d-dep-mtp.yaml @@ -0,0 +1,126 @@ +schema: 2 +name: "bs128-1p1d-dep-h200-fp8-mtp" + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.8.post1-cu130" + precision: "fp8" + +frontend: + nginx_container: nginx + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + # stream-interval: 50 + max-running-requests: 16 + + + # Prefill-specific mode + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + max-prefill-tokens: 163840 + chunked-prefill-size: 163840 + + # Request handling + load-balance-method: "round_robin" + + + decode: + nodes: 1 + workers: 1 + env: + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + + # KV cache and attention + attention-backend: "flashinfer" + + # Other flags + disable-radix-cache: true + stream-interval: 10 + + # Disagg + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.85 + max-running-requests: 192 + cuda-graph-max-bs: 192 + + # MTP settings + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "32x64x128x256x512" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs128-1p1d-dep-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs128-1p1d-dep-stp.yaml new file mode 100644 index 0000000000..bb0ddabd7f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs128-1p1d-dep-stp.yaml @@ -0,0 +1,117 @@ +schema: 2 +name: "bs128-1p1d-dep-h200-fp8" + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.8.post1-cu130" + precision: "fp8" + +frontend: + nginx_container: nginx + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + # stream-interval: 50 + watchdog-timeout: 1000000 + max-running-requests: 16 + + + # Prefill-specific mode + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + max-prefill-tokens: 163840 + chunked-prefill-size: 163840 + + # Request handling + load-balance-method: "round_robin" + + + decode: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + + # Parallelism + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + + # KV cache and attention + attention-backend: "flashinfer" + + # Other flags + disable-radix-cache: true + stream-interval: 10 + watchdog-timeout: 1000000 + + # Disagg + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.88 + max-running-requests: 256 + cuda-graph-max-bs: 256 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "64x128x256" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs16-1p3d-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs16-1p3d-mtp.yaml new file mode 100644 index 0000000000..1892e95fc1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs16-1p3d-mtp.yaml @@ -0,0 +1,124 @@ +schema: 2 +name: "bs16-1p3d-h200-fp8-mtp" + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.8.post1-cu130" + precision: "fp8" + +frontend: + nginx_container: nginx + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + # stream-interval: 50 + max-running-requests: 16 + + + # Prefill-specific mode + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.82 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + + # Request handling + load-balance-method: "round_robin" + + decode: + nodes: 3 + workers: 3 + env: + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Other flags + disable-radix-cache: true + stream-interval: 10 + + # Disagg + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.82 + max-running-requests: 32 + cuda-graph-max-bs: 32 + + # MTP settings + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4x8x16x32x64" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs16-1p3d-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs16-1p3d-stp.yaml new file mode 100644 index 0000000000..6562d0fa67 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs16-1p3d-stp.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: "bs16-1p3d-h200-fp8" + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.8.post1-cu130" + precision: "fp8" + +frontend: + nginx_container: nginx + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + # stream-interval: 50 + watchdog-timeout: 1000000 + max-running-requests: 16 + + + # Prefill-specific mode + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.82 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + + # Request handling + load-balance-method: "round_robin" + + decode: + nodes: 3 + workers: 3 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Other flags + disable-radix-cache: true + stream-interval: 10 + watchdog-timeout: 1000000 + + # Disagg + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.82 + max-running-requests: 32 + cuda-graph-max-bs: 32 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "8x16x32" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs4-1p7d-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs4-1p7d-mtp.yaml new file mode 100644 index 0000000000..a7a5d2c335 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs4-1p7d-mtp.yaml @@ -0,0 +1,124 @@ +schema: 2 +name: "bs4-1p7d-h200-fp8-mtp" + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.8.post1-cu130" + precision: "fp8" + +frontend: + nginx_container: nginx + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + # stream-interval: 50 + max-running-requests: 16 + + + # Prefill-specific mode + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.82 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + + # Request handling + load-balance-method: "round_robin" + + decode: + nodes: 7 + workers: 7 + env: + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Other flags + disable-radix-cache: true + stream-interval: 10 + + # Disagg + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.75 + max-running-requests: 2 + cuda-graph-max-bs: 2 + + # MTP settings + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x4x8" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs4-1p7d-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs4-1p7d-stp.yaml new file mode 100644 index 0000000000..78807ab23d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs4-1p7d-stp.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: "bs4-1p7d-h200-fp8" + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.8.post1-cu130" + precision: "fp8" + +frontend: + nginx_container: nginx + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + # stream-interval: 50 + watchdog-timeout: 1000000 + max-running-requests: 16 + + + # Prefill-specific mode + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.82 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + + # Request handling + load-balance-method: "round_robin" + + decode: + nodes: 7 + workers: 7 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Other flags + disable-radix-cache: true + stream-interval: 10 + watchdog-timeout: 1000000 + + # Disagg + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.82 + max-running-requests: 8 + cuda-graph-max-bs: 8 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x4x8" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs64-2p3d-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs64-2p3d-mtp.yaml new file mode 100644 index 0000000000..e356fa8cb0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs64-2p3d-mtp.yaml @@ -0,0 +1,133 @@ +schema: 2 +name: "bs64-2p3d-h200-fp8-mtp" + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.8.post1-cu130" + precision: "fp8" + +frontend: + nginx_container: nginx + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +engine: sglang +roles: + prefill: + nodes: 2 + workers: 2 + env: + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + # stream-interval: 50 + max-running-requests: 16 + + + # Prefill-specific mode + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.82 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + + # Request handling + load-balance-method: "round_robin" + + decode: + nodes: 3 + workers: 3 + env: + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Other flags + disable-radix-cache: true + stream-interval: 10 + + # Disagg + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + context-length: 72000 + max-total-tokens: 128000 + # Memory and token limits + mem-fraction-static: 0.75 + max-running-requests: 16 + cuda-graph-max-bs: 16 + + # MTP settings + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "32x64x128" + req_rate: "inf" + +# benchmark: +# type: "gpqa" +# num_examples: 198 +# repeat: 4 +# num_threads: 32 +# max_tokens: 64000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs64-2p3d-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs64-2p3d-stp.yaml new file mode 100644 index 0000000000..781d346906 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs64-2p3d-stp.yaml @@ -0,0 +1,124 @@ +schema: 2 +name: "bs64-2p3d-h200-fp8" + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.8.post1-cu130" + precision: "fp8" + +frontend: + nginx_container: nginx + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +engine: sglang +roles: + prefill: + nodes: 2 + workers: 2 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + # stream-interval: 50 + watchdog-timeout: 1000000 + max-running-requests: 16 + + + # Prefill-specific mode + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.82 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + + # Request handling + load-balance-method: "round_robin" + + decode: + nodes: 3 + workers: 3 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Other flags + disable-radix-cache: true + stream-interval: 10 + watchdog-timeout: 1000000 + + # Disagg + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + #context-length: 72000 + # max-total-tokens: 128000 + # Memory and token limits + mem-fraction-static: 0.82 + max-running-requests: 128 + cuda-graph-max-bs: 128 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "32x64x128" + req_rate: "inf" + +# benchmark: +# type: "gpqa" +# num_examples: 198 +# repeat: 4 +# num_threads: 32 +# max_tokens: 64000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs8-1p6d-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs8-1p6d-mtp.yaml new file mode 100644 index 0000000000..17c0d6e297 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs8-1p6d-mtp.yaml @@ -0,0 +1,125 @@ +schema: 2 +name: "bs8-1p6d-h200-fp8-mtp" + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.8.post1-cu130" + precision: "fp8" + +frontend: + nginx_container: nginx + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + # Decode-specific environment variables + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + # stream-interval: 50 + max-running-requests: 16 + + + # Prefill-specific mode + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.82 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + + # Request handling + load-balance-method: "round_robin" + + + decode: + nodes: 6 + workers: 6 + env: + SGLANG_ENABLE_SPEC_V2: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + watchdog-timeout: 1000000 + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Other flags + disable-radix-cache: true + stream-interval: 10 + + # Disagg + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.82 + max-running-requests: 16 + cuda-graph-max-bs: 16 + + # MTP settings + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2x4x8x16x32" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs8-1p6d-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs8-1p6d-stp.yaml new file mode 100644 index 0000000000..11afe7a270 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs8-1p6d-stp.yaml @@ -0,0 +1,116 @@ +schema: 2 +name: "bs8-1p6d-h200-fp8" + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.8.post1-cu130" + precision: "fp8" + +frontend: + nginx_container: nginx + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +dynamo: + # Dynamo 0.8.0 was the default when this recipe was written; pin it explicitly. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" + + source: + pypi: "0.8.0" +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Radix cache disabled + disable-radix-cache: true + + # Other flags + # stream-interval: 50 + watchdog-timeout: 1000000 + max-running-requests: 16 + + + # Prefill-specific mode + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.82 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + + # Request handling + load-balance-method: "round_robin" + + + decode: + nodes: 6 + workers: 6 + env: + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + + args: + # Model configuration + served-model-name: "deepseek-ai/DeepSeek-R1" + model-path: "/model/" + skip-tokenizer-init: true + trust-remote-code: true + + # Parallelism + tp-size: 8 + dp-size: 1 + ep-size: 1 + + # KV cache and attention + attention-backend: "flashinfer" + + # Other flags + disable-radix-cache: true + stream-interval: 10 + watchdog-timeout: 1000000 + + # Disagg + disaggregation-bootstrap-port: 30001 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.82 + max-running-requests: 16 + cuda-graph-max-bs: 16 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4x8x16" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p1d-dep8-b8-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p1d-dep8-b8-eplb0-mtp3.yaml new file mode 100644 index 0000000000..f24a9de07f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p1d-dep8-b8-eplb0-mtp3.yaml @@ -0,0 +1,120 @@ +schema: 2 +name: "ctx1_gen1_dep8_batch8_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 1 + + workers: 1 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 5 + - 6 + - 7 + - 8 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "90" + req_rate: "inf" + +frontend: + nginx_container: "nginx-sqsh" + type: "dynamo" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp3.yaml new file mode 100644 index 0000000000..40cdb3c74b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp3.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: "ctx1_gen3_tep8_batch16_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 3 + + workers: 3 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 16 + max_num_tokens: 64 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "66" + req_rate: "inf" + +frontend: + nginx_container: "nginx-sqsh" + type: "dynamo" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b1-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b1-eplb0-mtp0.yaml new file mode 100644 index 0000000000..4819f17f8e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b1-eplb0-mtp0.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "ctx1_gen5_tep8_batch1_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + + decode: + nodes: 5 + + workers: 5 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "6" + req_rate: "inf" + +frontend: + nginx_container: "nginx-sqsh" + type: "dynamo" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b1-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b1-eplb0-mtp3.yaml new file mode 100644 index 0000000000..8c46f69689 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b1-eplb0-mtp3.yaml @@ -0,0 +1,116 @@ +schema: 2 +name: "ctx1_gen5_tep8_batch1_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 5 + + workers: 5 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "6" + req_rate: "inf" + +frontend: + nginx_container: "nginx-sqsh" + type: "dynamo" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b16-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b16-eplb0-mtp0.yaml new file mode 100644 index 0000000000..9d01d61c99 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b16-eplb0-mtp0.yaml @@ -0,0 +1,124 @@ +schema: 2 +name: "ctx1_gen5_tep8_batch8_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + + decode: + nodes: 5 + + workers: 5 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 12 + - 13 + - 14 + - 15 + - 16 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "10x15x25x50x100" + req_rate: "inf" + +frontend: + nginx_container: "nginx-sqsh" + type: "dynamo" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b8-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b8-eplb0-mtp3.yaml new file mode 100644 index 0000000000..05dd640962 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b8-eplb0-mtp3.yaml @@ -0,0 +1,123 @@ +schema: 2 +name: "ctx1_gen5_tep8_batch8_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 5 + + workers: 5 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "10x15x30x60" + req_rate: "inf" + +frontend: + nginx_container: "nginx-sqsh" + type: "dynamo" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-2p5d-tep8-b64-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-2p5d-tep8-b64-eplb0-mtp0.yaml new file mode 100644 index 0000000000..ccdabf7309 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-2p5d-tep8-b64-eplb0-mtp0.yaml @@ -0,0 +1,119 @@ +schema: 2 +name: "ctx2_gen5_tep8_batch64_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + + decode: + nodes: 5 + + workers: 5 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 64 + max_num_tokens: 64 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 58 + - 60 + - 62 + - 64 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "370" + req_rate: "inf" + +frontend: + nginx_container: "nginx-sqsh" + type: "dynamo" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp3.yaml new file mode 100644 index 0000000000..5029cd5517 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp3.yaml @@ -0,0 +1,124 @@ +schema: 2 +name: "ctx3_gen1_dep8_batch64_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 3 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 1 + + workers: 1 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 64 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 48 + - 56 + - 60 + - 62 + - 64 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "548" + req_rate: "inf" + +frontend: + nginx_container: "nginx-sqsh" + type: "dynamo" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-4p1d-dep8-b192-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-4p1d-dep8-b192-eplb0-mtp0.yaml new file mode 100644 index 0000000000..f153ec6020 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-4p1d-dep8-b192-eplb0-mtp0.yaml @@ -0,0 +1,122 @@ +schema: 2 +name: "ctx4_gen1_dep8_batch192_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 4 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + + decode: + nodes: 1 + + workers: 1 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 192 + max_num_tokens: 192 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 152 + - 160 + - 168 + - 176 + - 184 + - 190 + - 192 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1606" + req_rate: "inf" + +frontend: + nginx_container: "nginx-sqsh" + type: "dynamo" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-4p3d-dep8-b32-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-4p3d-dep8-b32-eplb0-mtp0.yaml new file mode 100644 index 0000000000..def3f8dbab --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-4p3d-dep8-b32-eplb0-mtp0.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: "ctx4_gen3_dep8_batch32_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 4 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + + decode: + nodes: 3 + + workers: 3 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 28 + - 30 + - 32 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "837" + req_rate: "inf" + +frontend: + nginx_container: "nginx-sqsh" + type: "dynamo" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-5p1d-dep8-b192-eplb0-mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-5p1d-dep8-b192-eplb0-mtp1.yaml new file mode 100644 index 0000000000..c4df00f6a7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-5p1d-dep8-b192-eplb0-mtp1.yaml @@ -0,0 +1,128 @@ +schema: 2 +name: "ctx5_gen1_dep8_batch192_eplb0_mtp1" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 3 + workers: 5 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + + decode: + nodes: 1 + + workers: 1 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 192 + max_num_tokens: 384 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 130 + - 132 + - 134 + - 136 + - 138 + - 168 + - 192 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1096x1691" + req_rate: "inf" + +frontend: + nginx_container: "nginx-sqsh" + type: "dynamo" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-5p2d-dep8-b32-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-5p2d-dep8-b32-eplb0-mtp3.yaml new file mode 100644 index 0000000000..2cd5e90785 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-5p2d-dep8-b32-eplb0-mtp3.yaml @@ -0,0 +1,123 @@ +schema: 2 +name: "ctx5_gen2_dep8_batch32_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 3 + workers: 5 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 2 + + workers: 2 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 20 + - 24 + - 28 + - 30 + - 32 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "658" + req_rate: "inf" + +frontend: + nginx_container: "nginx-sqsh" + type: "dynamo" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-7p2d-dep8-b128-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-7p2d-dep8-b128-eplb0-mtp0.yaml new file mode 100644 index 0000000000..825cee57c7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-7p2d-dep8-b128-eplb0-mtp0.yaml @@ -0,0 +1,118 @@ +schema: 2 +name: "ctx7_gen2_dep8_batch128_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 4 + workers: 7 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + + decode: + nodes: 2 + + workers: 2 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 128 + max_num_tokens: 128 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 116 + - 120 + - 124 + - 128 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2222" + req_rate: "inf" + +frontend: + nginx_container: "nginx-sqsh" + type: "dynamo" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p1d-dep8-b128-eplb0-mtp0-c128.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p1d-dep8-b128-eplb0-mtp0-c128.yaml new file mode 100644 index 0000000000..9fb9623a22 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p1d-dep8-b128-eplb0-mtp0-c128.yaml @@ -0,0 +1,119 @@ +schema: 2 +name: ctx1_gen1_dep8_batch128_eplb0_mtp0_128 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.2 + max_batch_size: 1 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 8 + + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + disable_overlap_scheduler: false + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 128 + max_num_tokens: 512 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [128] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p1d-dep8-b256-eplb0-mtp0-c256.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p1d-dep8-b256-eplb0-mtp0-c256.yaml new file mode 100644 index 0000000000..ed5e7e0bfd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p1d-dep8-b256-eplb0-mtp0-c256.yaml @@ -0,0 +1,119 @@ +schema: 2 +name: ctx1_gen1_dep8_batch256_eplb0_mtp0_256 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.2 + max_batch_size: 1 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 8 + + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + disable_overlap_scheduler: false + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 256 + max_num_tokens: 512 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [256] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p1d-tp8-b1-eplb0-mtp0-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p1d-tp8-b1-eplb0-mtp0-c1.yaml new file mode 100644 index 0000000000..dae549139f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p1d-tp8-b1-eplb0-mtp0-c1.yaml @@ -0,0 +1,121 @@ +schema: 2 +name: ctx1_gen1_tp8_batch1_eplb0_mtp0_1 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + disable_overlap_scheduler: true + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 64 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 8 + + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 1 + disable_overlap_scheduler: false + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [1] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p2d-dep8-b64-eplb0-mtp0-c128.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p2d-dep8-b64-eplb0-mtp0-c128.yaml new file mode 100644 index 0000000000..db313db678 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p2d-dep8-b64-eplb0-mtp0-c128.yaml @@ -0,0 +1,119 @@ +schema: 2 +name: ctx1_gen2_dep8_batch64_eplb0_mtp0_128 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.2 + max_batch_size: 1 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 8 + + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + disable_overlap_scheduler: false + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 64 + max_num_tokens: 512 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [128] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p2d-tp8-b4-eplb0-mtp3-c8.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p2d-tp8-b4-eplb0-mtp3-c8.yaml new file mode 100644 index 0000000000..7bf1ad646e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p2d-tp8-b4-eplb0-mtp3-c8.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: ctx1_gen2_tp8_batch32_eplb0_mtp3_8 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.2 + max_batch_size: 1 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 8 + + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + attention_dp_config: + enable_balance: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + disable_overlap_scheduler: false + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [8] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p4d-tp8-b16-eplb0-mtp3-c64.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p4d-tp8-b16-eplb0-mtp3-c64.yaml new file mode 100644 index 0000000000..1237c15e29 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p4d-tp8-b16-eplb0-mtp3-c64.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: ctx1_gen4_tp8_batch16_eplb0_mtp3_64 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.2 + max_batch_size: 1 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 8 + + + decode: + nodes: 4 + workers: 4 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + attention_dp_config: + enable_balance: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + disable_overlap_scheduler: false + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 16 + max_num_tokens: 64 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [64] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p4d-tp8-b32-eplb0-mtp0-c128.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p4d-tp8-b32-eplb0-mtp0-c128.yaml new file mode 100644 index 0000000000..1487f687f0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p4d-tp8-b32-eplb0-mtp0-c128.yaml @@ -0,0 +1,120 @@ +schema: 2 +name: ctx1_gen4_tp8_batch32_eplb0_mtp0_128 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.2 + max_batch_size: 1 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 8 + + + decode: + nodes: 4 + workers: 4 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + disable_overlap_scheduler: false + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 32 + max_num_tokens: 512 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [128] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p4d-tp8-b8-eplb0-mtp0-c32.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p4d-tp8-b8-eplb0-mtp0-c32.yaml new file mode 100644 index 0000000000..d1acbd3a09 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p4d-tp8-b8-eplb0-mtp0-c32.yaml @@ -0,0 +1,120 @@ +schema: 2 +name: ctx1_gen4_tp8_batch32_eplb0_mtp0_32 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.2 + max_batch_size: 1 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 8 + + + decode: + nodes: 4 + workers: 4 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + disable_overlap_scheduler: false + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 8 + max_num_tokens: 8 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [32] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p6d-tp8-b1-eplb0-mtp3-c8.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p6d-tp8-b1-eplb0-mtp3-c8.yaml new file mode 100644 index 0000000000..b2e1007e18 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p6d-tp8-b1-eplb0-mtp3-c8.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: ctx1_gen6_tp8_batch8_eplb0_mtp3_8 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.2 + max_batch_size: 1 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 8 + + + decode: + nodes: 6 + workers: 6 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + attention_dp_config: + enable_balance: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 1 + disable_overlap_scheduler: false + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [8] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p6d-tp8-b16-eplb0-mtp0-c96.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p6d-tp8-b16-eplb0-mtp0-c96.yaml new file mode 100644 index 0000000000..dfee643208 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p6d-tp8-b16-eplb0-mtp0-c96.yaml @@ -0,0 +1,120 @@ +schema: 2 +name: ctx1_gen6_tp8_batch16_eplb0_mtp0_96 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.2 + max_batch_size: 1 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 8 + + + decode: + nodes: 6 + workers: 6 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + disable_overlap_scheduler: false + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 16 + max_num_tokens: 512 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [96] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p6d-tp8-b8-eplb0-mtp3-c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p6d-tp8-b8-eplb0-mtp3-c48.yaml new file mode 100644 index 0000000000..3b0cfdd01a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p6d-tp8-b8-eplb0-mtp3-c48.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: ctx1_gen6_tp8_batch8_eplb0_mtp3_48 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.2 + max_batch_size: 1 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 8 + + + decode: + nodes: 6 + workers: 6 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + attention_dp_config: + enable_balance: true + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + disable_overlap_scheduler: false + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [48] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-2p1d-dep8-b32-eplb0-mtp3-c288.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-2p1d-dep8-b32-eplb0-mtp3-c288.yaml new file mode 100644 index 0000000000..6d2e644506 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-2p1d-dep8-b32-eplb0-mtp3-c288.yaml @@ -0,0 +1,129 @@ +schema: 2 +name: ctx2_gen1_dep8_batch32_eplb0_mtp3_288 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.2 + max_batch_size: 1 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 8 + + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + disable_overlap_scheduler: false + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 32 + max_num_tokens: 1024 + max_seq_len: 9344 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [288] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-2p1d-dep8-b640-eplb0-mtp0-c640.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-2p1d-dep8-b640-eplb0-mtp0-c640.yaml new file mode 100644 index 0000000000..ede8d2ec53 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-2p1d-dep8-b640-eplb0-mtp0-c640.yaml @@ -0,0 +1,119 @@ +schema: 2 +name: ctx2_gen1_dep8_batch640_eplb0_mtp0_640 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.2 + max_batch_size: 1 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 8 + + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 640 + disable_overlap_scheduler: false + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 640 + max_num_tokens: 512 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [640] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-2p3d-dep8-b8-eplb0-mtp3-c224.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-2p3d-dep8-b8-eplb0-mtp3-c224.yaml new file mode 100644 index 0000000000..97103cac7c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-2p3d-dep8-b8-eplb0-mtp3-c224.yaml @@ -0,0 +1,129 @@ +schema: 2 +name: ctx2_gen3_dep8_batch8_eplb0_mtp3_224 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.2 + max_batch_size: 1 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 8 + + + decode: + nodes: 3 + workers: 3 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 + cache_transceiver_config: + backend: DEFAULT + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + disable_overlap_scheduler: false + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 8 + max_num_tokens: 256 + max_seq_len: 9344 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [224] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-4p1d-dep8-b128-eplb0-mtp2-c1088.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-4p1d-dep8-b128-eplb0-mtp2-c1088.yaml new file mode 100644 index 0000000000..f313d59a4b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-4p1d-dep8-b128-eplb0-mtp2-c1088.yaml @@ -0,0 +1,129 @@ +schema: 2 +name: ctx4_gen1_dep8_batch128_eplb0_mtp2_1088 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 4 + workers: 4 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + max_batch_size: 1 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 2 + tensor_parallel_size: 8 + + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_RNDV_SCHEME: "put_zcopy" + + args: + allreduce_strategy: AUTO + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + disable_overlap_scheduler: false + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 128 + max_num_tokens: 3072 + max_seq_len: 9344 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 2 + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [1088] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp1.yaml new file mode 100644 index 0000000000..b770fc7e2b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp1.yaml @@ -0,0 +1,133 @@ +schema: 2 +name: "ctx10_gen1_dep8_batch256_eplb0_mtp1" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 3 + workers: 10 + gpus: 2 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + + decode: + nodes: 1 + + workers: 1 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 256 + max_num_tokens: 512 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2198" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p3d-tep4-b32-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p3d-tep4-b32-eplb0-mtp0.yaml new file mode 100644 index 0000000000..a944acc858 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p3d-tep4-b32-eplb0-mtp0.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: "ctx1_gen3_tep4_batch32_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + + decode: + nodes: 2 + workers: 3 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "105" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p3d-tep8-b1-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p3d-tep8-b1-eplb0-mtp0.yaml new file mode 100644 index 0000000000..43293b8635 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p3d-tep8-b1-eplb0-mtp0.yaml @@ -0,0 +1,134 @@ +schema: 2 +name: "ctx1_gen3_tep8_batch1_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + + decode: + nodes: 3 + + workers: 3 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 768 + - 1024 + - 2048 + - 1 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0.yaml new file mode 100644 index 0000000000..8298e611d9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0.yaml @@ -0,0 +1,125 @@ +schema: 2 +name: "ctx1_gen3_tep8_batch16_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + + decode: + nodes: 3 + + workers: 3 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "63" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep4-b2-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep4-b2-eplb0-mtp0.yaml new file mode 100644 index 0000000000..c468b552c8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep4-b2-eplb0-mtp0.yaml @@ -0,0 +1,122 @@ +schema: 2 +name: "ctx1_gen4_tep4_batch2_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + + decode: + nodes: 2 + + workers: 4 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 2 + max_num_tokens: 2 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "12" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep4-b8-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep4-b8-eplb0-mtp3.yaml new file mode 100644 index 0000000000..6e6e3f4339 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep4-b8-eplb0-mtp3.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: "ctx1_gen4_tep4_batch8_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 2 + + workers: 4 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + gpus_per_node: 4 + allreduce_strategy: MNNVL + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "52" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3.yaml new file mode 100644 index 0000000000..fdadfd7af9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: "ctx1_gen4_tep8_batch1_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 4 + + workers: 4 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "8" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3.yaml new file mode 100644 index 0000000000..6ebfa26856 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3.yaml @@ -0,0 +1,130 @@ +schema: 2 +name: "ctx1_gen4_tep8_batch4_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 4 + + workers: 4 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 3 + - 4 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "32" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-3p1d-dep8-b16-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-3p1d-dep8-b16-eplb0-mtp3.yaml new file mode 100644 index 0000000000..17806d7cd7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-3p1d-dep8-b16-eplb0-mtp3.yaml @@ -0,0 +1,129 @@ +schema: 2 +name: "ctx3_gen1_dep8_batch16_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 3 + gpus: 2 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 1 + + workers: 1 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 16 + max_num_tokens: 64 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "181" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-5p2d-dep8-b32-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-5p2d-dep8-b32-eplb0-mtp0.yaml new file mode 100644 index 0000000000..def8f83d7d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-5p2d-dep8-b32-eplb0-mtp0.yaml @@ -0,0 +1,124 @@ +schema: 2 +name: "ctx5_gen2_dep8_batch32_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 5 + gpus: 2 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + + decode: + nodes: 2 + + workers: 2 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "589" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-6p1d-dep8-b128-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-6p1d-dep8-b128-eplb0-mtp0.yaml new file mode 100644 index 0000000000..d9432e68c0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-6p1d-dep8-b128-eplb0-mtp0.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: "ctx6_gen1_dep8_batch128_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 6 + gpus: 2 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + + decode: + nodes: 1 + + workers: 1 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 128 + max_num_tokens: 128 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 768 + - 1024 + - 2048 + - 128 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1093" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-8p1d-dep8-b256-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-8p1d-dep8-b256-eplb0-mtp0.yaml new file mode 100644 index 0000000000..f4cde2cac2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-8p1d-dep8-b256-eplb0-mtp0.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: "ctx8_gen1_dep8_batch256_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 8 + gpus: 2 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + + decode: + nodes: 1 + + workers: 1 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 256 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 768 + - 1024 + - 2048 + - 256 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-9p1d-dep8-b128-eplb0-mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-9p1d-dep8-b128-eplb0-mtp1.yaml new file mode 100644 index 0000000000..7ee89a73a8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-9p1d-dep8-b128-eplb0-mtp1.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: "ctx9_gen1_dep8_batch128_eplb0_mtp1" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 3 + workers: 9 + gpus: 2 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + + decode: + nodes: 1 + + workers: 1 + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 128 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8448 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1197" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p1d-dp8-b8-eplb0-mtp3-c72.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p1d-dp8-b8-eplb0-mtp3-c72.yaml new file mode 100644 index 0000000000..a2ce5bd5bf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p1d-dp8-b8-eplb0-mtp3-c72.yaml @@ -0,0 +1,137 @@ +schema: 2 +name: ctx1_gen1_dp8_batch8_eplb0_mtp3_72 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + max_batch_size: 8 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 4 + + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + disable_overlap_scheduler: false + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 8 + max_num_tokens: 90 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [72] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p2d-tp8-b16-eplb0-mtp3-c40.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p2d-tp8-b16-eplb0-mtp3-c40.yaml new file mode 100644 index 0000000000..878b0e29da --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p2d-tp8-b16-eplb0-mtp3-c40.yaml @@ -0,0 +1,138 @@ +schema: 2 +name: ctx1_gen2_tp8_batch16_eplb0_mtp3_40 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + max_batch_size: 8 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 4 + + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + disable_overlap_scheduler: false + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 16 + max_num_tokens: 80 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [40] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p4d-tp8-b1-eplb0-mtp3-c8.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p4d-tp8-b1-eplb0-mtp3-c8.yaml new file mode 100644 index 0000000000..5e6f22c9d6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p4d-tp8-b1-eplb0-mtp3-c8.yaml @@ -0,0 +1,138 @@ +schema: 2 +name: ctx1_gen4_tp8_batch1_eplb0_mtp3_8 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + max_batch_size: 8 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 4 + + + decode: + nodes: 4 + workers: 4 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 1 + disable_overlap_scheduler: false + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [5] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p4d-tp8-b16-eplb0-mtp0-c64.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p4d-tp8-b16-eplb0-mtp0-c64.yaml new file mode 100644 index 0000000000..268e130912 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p4d-tp8-b16-eplb0-mtp0-c64.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: ctx1_gen4_tp8_batch16_eplb0_mtp0_64 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + max_batch_size: 8 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + + + decode: + nodes: 4 + workers: 4 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + disable_overlap_scheduler: false + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 16 + max_num_tokens: 512 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [64] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p4d-tp8-b4-eplb0-mtp3-c20.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p4d-tp8-b4-eplb0-mtp3-c20.yaml new file mode 100644 index 0000000000..7e8eb18108 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p4d-tp8-b4-eplb0-mtp3-c20.yaml @@ -0,0 +1,138 @@ +schema: 2 +name: ctx1_gen4_tp8_batch4_eplb0_mtp3_20 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + max_batch_size: 8 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 4 + + + decode: + nodes: 4 + workers: 4 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + disable_overlap_scheduler: false + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 4 + max_num_tokens: 20 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [20] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p8d-tp8-b1-eplb0-mtp0-c16.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p8d-tp8-b1-eplb0-mtp0-c16.yaml new file mode 100644 index 0000000000..6ae0592cf4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p8d-tp8-b1-eplb0-mtp0-c16.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: ctx1_gen8_tp8_batch2_eplb0_mtp0_16 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + max_batch_size: 8 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + + + decode: + nodes: 8 + workers: 8 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 1 + disable_overlap_scheduler: false + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [10] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-2p1d-dp8-b16-eplb0-mtp3-c144.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-2p1d-dp8-b16-eplb0-mtp3-c144.yaml new file mode 100644 index 0000000000..1973754c90 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-2p1d-dp8-b16-eplb0-mtp3-c144.yaml @@ -0,0 +1,137 @@ +schema: 2 +name: ctx2_gen1_dp8_batch16_eplb0_mtp3_144 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + max_batch_size: 8 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 4 + + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + disable_overlap_scheduler: false + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 16 + max_num_tokens: 180 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [144] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-2p1d-dp8-b32-eplb0-mtp0-c256.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-2p1d-dp8-b32-eplb0-mtp0-c256.yaml new file mode 100644 index 0000000000..adc7a95d4a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-2p1d-dp8-b32-eplb0-mtp0-c256.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: ctx2_gen1_dp8_batch32_eplb0_mtp0_256 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + max_batch_size: 8 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + disable_overlap_scheduler: false + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 32 + max_num_tokens: 512 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [256] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-3p1d-dp8-b64-eplb0-mtp0-c512.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-3p1d-dp8-b64-eplb0-mtp0-c512.yaml new file mode 100644 index 0000000000..6185f89f18 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-3p1d-dp8-b64-eplb0-mtp0-c512.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: ctx3_gen1_dp8_batch64_eplb0_mtp0_512 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 3 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + max_batch_size: 8 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + disable_overlap_scheduler: false + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 64 + max_num_tokens: 512 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [512] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-3p5d-tp8-b64-eplb0-mtp0-c256.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-3p5d-tp8-b64-eplb0-mtp0-c256.yaml new file mode 100644 index 0000000000..9fa54f9385 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-3p5d-tp8-b64-eplb0-mtp0-c256.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: ctx3_gen5_tp8_batch64_eplb0_mtp0_256 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 3 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + max_batch_size: 8 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + + + decode: + nodes: 5 + workers: 5 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + disable_overlap_scheduler: false + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 64 + max_num_tokens: 512 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [256] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-4p1d-dp8-b64-eplb0-mtp2-c512.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-4p1d-dp8-b64-eplb0-mtp2-c512.yaml new file mode 100644 index 0000000000..5b8f68984e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-4p1d-dp8-b64-eplb0-mtp2-c512.yaml @@ -0,0 +1,137 @@ +schema: 2 +name: ctx4_gen1_dp8_batch64_eplb0_mtp2_512 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 4 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + max_batch_size: 8 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 2 + tensor_parallel_size: 4 + + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + disable_overlap_scheduler: false + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 64 + max_num_tokens: 650 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 2 + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [512] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-5p1d-dp8-b128-eplb0-mtp0-c1075.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-5p1d-dp8-b128-eplb0-mtp0-c1075.yaml new file mode 100644 index 0000000000..9c8a8e8b10 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-5p1d-dp8-b128-eplb0-mtp0-c1075.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: ctx5_gen1_dp8_batch128_eplb0_mtp0_1075 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 3 + workers: 5 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + max_batch_size: 8 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + disable_overlap_scheduler: false + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 128 + max_num_tokens: 512 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [1075] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-7p1d-dep8-b384-eplb0-mtp0-c3072.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-7p1d-dep8-b384-eplb0-mtp0-c3072.yaml new file mode 100644 index 0000000000..2687b56fbb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-7p1d-dep8-b384-eplb0-mtp0-c3072.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: ctx7_gen1_dep8_batch384_eplb0_mtp0_3072 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 4 + workers: 7 + gpus: 4 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: false + disable_overlap_scheduler: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + max_batch_size: 8 + max_num_tokens: 8320 + max_seq_len: 8320 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + NCCL_GRAPH_MIXING_SUPPORT: "0" + OMPI_MCA_coll_ucc_enable: "0" + TLLM_ALL_RANK_LOG: "1" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + UCX_MAX_RMA_RAILS: "1" + UCX_MAX_RNDV_RAILS: "1" + UCX_RNDV_SCHEME: "put_zcopy" + OMPI_MCA_btl: "tcp,self" + OMPI_MCA_pml: "ob1" + TRTLLM_UCX_INTERFACE: "mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1,mlx5_16:1,mlx5_17:1,mlx5_20:1,mlx5_21:1,mlx5_22:1,mlx5_23:1,mlx5_4:1,mlx5_5:1,mlx5_8:1,mlx5_9:1,mlx5_2:1,mlx5_3:1" + + args: + allreduce_strategy: AUTO + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8320 + cuda_graph_config: + enable_padding: true + max_batch_size: 384 + disable_overlap_scheduler: false + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 384 + max_num_tokens: 512 + max_seq_len: 9344 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 20 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: [3072] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-10p1d-dep16-b256-eplb256-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-10p1d-dep16-b256-eplb256-mtp0.yaml new file mode 100644 index 0000000000..dc011cc889 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-10p1d-dep16-b256-eplb256-mtp0.yaml @@ -0,0 +1,157 @@ +schema: 2 +name: "ctx10_gen1_dep16_batch256_eplb256_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 10 + workers: 10 + + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + + decode: + nodes: 4 + + workers: 1 + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 256 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + load_balancer: + num_slots: 256 + layer_updates_per_iter: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-11p1d-dep16-b256-eplb256-mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-11p1d-dep16-b256-eplb256-mtp1.yaml new file mode 100644 index 0000000000..4975070891 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-11p1d-dep16-b256-eplb256-mtp1.yaml @@ -0,0 +1,163 @@ +schema: 2 +name: "ctx11_gen1_dep16_batch256_eplb256_mtp1" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 11 + workers: 11 + + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + + decode: + nodes: 4 + + workers: 1 + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 256 + max_num_tokens: 512 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + load_balancer: + num_slots: 256 + layer_updates_per_iter: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4301" + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml new file mode 100644 index 0000000000..901eec3517 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml @@ -0,0 +1,123 @@ +schema: 2 +name: "ctx1_gen4_tep8_batch1_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + + decode: + nodes: 8 + + workers: 4 + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "5" + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b16-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b16-eplb0-mtp0.yaml new file mode 100644 index 0000000000..7e70d201cf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b16-eplb0-mtp0.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: "ctx1_gen4_tep8_batch16_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + + decode: + nodes: 8 + + workers: 4 + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 6 + - 8 + - 9 + - 10 + - 11 + - 14 + - 15 + - 16 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "12x44x76" + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp3.yaml new file mode 100644 index 0000000000..4b0a1d57f3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp3.yaml @@ -0,0 +1,134 @@ +schema: 2 +name: "ctx1_gen4_tep8_batch8_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 8 + + workers: 4 + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4x8x12x24x48" + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0.yaml new file mode 100644 index 0000000000..73ac196bfd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0.yaml @@ -0,0 +1,123 @@ +schema: 2 +name: "ctx2_gen1_dep32_batch8_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + + decode: + nodes: 8 + + workers: 1 + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 8 + max_num_tokens: 8 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + moe_config: + backend: WIDEEP + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "333" + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-3p1d-dep32-b4-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-3p1d-dep32-b4-eplb0-mtp3.yaml new file mode 100644 index 0000000000..2236c57409 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-3p1d-dep32-b4-eplb0-mtp3.yaml @@ -0,0 +1,128 @@ +schema: 2 +name: "ctx3_gen1_dep32_batch4_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 3 + workers: 3 + + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 8 + + workers: 1 + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "180" + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep16-b64-eplb256-mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep16-b64-eplb256-mtp1.yaml new file mode 100644 index 0000000000..6f0ea09e6a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep16-b64-eplb256-mtp1.yaml @@ -0,0 +1,139 @@ +schema: 2 +name: "ctx7_gen1_dep16_batch64_eplb256_mtp1" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 7 + workers: 7 + + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + + decode: + nodes: 4 + + workers: 1 + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 64 + max_num_tokens: 128 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + load_balancer: + num_slots: 256 + layer_updates_per_iter: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1229" + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep32-b32-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep32-b32-eplb0-mtp0.yaml new file mode 100644 index 0000000000..e0d91dded0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep32-b32-eplb0-mtp0.yaml @@ -0,0 +1,126 @@ +schema: 2 +name: "ctx7_gen1_dep32_batch32_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 7 + workers: 7 + + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + + decode: + nodes: 8 + + workers: 1 + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1229" + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep16-b128-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep16-b128-eplb0-mtp0.yaml new file mode 100644 index 0000000000..0a04a36eea --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep16-b128-eplb0-mtp0.yaml @@ -0,0 +1,138 @@ +schema: 2 +name: "ctx8_gen1_dep16_batch128_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 8 + workers: 8 + + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + + decode: + nodes: 4 + + workers: 1 + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 128 + max_num_tokens: 128 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2253" + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep32-b16-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep32-b16-eplb0-mtp3.yaml new file mode 100644 index 0000000000..013950ed79 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep32-b16-eplb0-mtp3.yaml @@ -0,0 +1,130 @@ +schema: 2 +name: "ctx8_gen1_dep32_batch16_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 8 + workers: 8 + + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 8 + + workers: 1 + env: + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 16 + max_num_tokens: 64 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "666" + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b1-eplb0-mtp0-c6.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b1-eplb0-mtp0-c6.yaml new file mode 100644 index 0000000000..c25597ce46 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b1-eplb0-mtp0-c6.yaml @@ -0,0 +1,125 @@ +schema: 2 +name: ctx1_gen3_tep8_batch1_eplb0_mtp0_6 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 8 + + + decode: + nodes: 6 + workers: 3 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + allreduce_strategy: MNNVL + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['6'] + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0-c63.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0-c63.yaml new file mode 100644 index 0000000000..64151a38e8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0-c63.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: ctx1_gen3_tep8_batch16_eplb0_mtp0_63 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 8 + + + decode: + nodes: 6 + workers: 3 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + allreduce_strategy: MNNVL + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['63'] + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b2-eplb0-mtp3-c6.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b2-eplb0-mtp3-c6.yaml new file mode 100644 index 0000000000..e3f15d2954 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b2-eplb0-mtp3-c6.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: ctx1_gen3_tep8_batch2_eplb0_mtp3_6 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 8 + + + decode: + nodes: 6 + workers: 3 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + allreduce_strategy: MNNVL + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 2 + max_num_tokens: 8 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 100 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['6'] + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp0-c18.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp0-c18.yaml new file mode 100644 index 0000000000..db3309c1d7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp0-c18.yaml @@ -0,0 +1,125 @@ +schema: 2 +name: ctx1_gen3_tep8_batch4_eplb0_mtp0_18 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 8 + + + decode: + nodes: 6 + workers: 3 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + allreduce_strategy: MNNVL + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 4 + max_num_tokens: 4 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['18'] + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp3-c15.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp3-c15.yaml new file mode 100644 index 0000000000..bd33d17265 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp3-c15.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: ctx1_gen3_tep8_batch4_eplb0_mtp3_15 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 8 + + + decode: + nodes: 6 + workers: 3 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + allreduce_strategy: MNNVL + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 100 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['15'] + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b2-eplb0-mtp3-c90.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b2-eplb0-mtp3-c90.yaml new file mode 100644 index 0000000000..e78a95ad6d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b2-eplb0-mtp3-c90.yaml @@ -0,0 +1,130 @@ +schema: 2 +name: ctx2_gen1_dep32_batch2_eplb0_mtp3_90 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 8 + + + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 8 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 100 + tensor_parallel_size: 32 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['90'] + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0-c333.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0-c333.yaml new file mode 100644 index 0000000000..49da3dc98c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0-c333.yaml @@ -0,0 +1,125 @@ +schema: 2 +name: ctx2_gen1_dep32_batch8_eplb0_mtp0_333 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 8 + + + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 8 + max_num_tokens: 8 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 32 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['333'] + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b16-eplb0-mtp3-c333.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b16-eplb0-mtp3-c333.yaml new file mode 100644 index 0000000000..9b8f24aaaa --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b16-eplb0-mtp3-c333.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: ctx3_gen1_dep16_batch16_eplb0_mtp3_333 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 6 + workers: 3 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 8 + + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 16 + max_num_tokens: 64 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 100 + tensor_parallel_size: 16 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['333'] + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b32-eplb0-mtp0-c615.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b32-eplb0-mtp0-c615.yaml new file mode 100644 index 0000000000..2ff39afc48 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b32-eplb0-mtp0-c615.yaml @@ -0,0 +1,128 @@ +schema: 2 +name: ctx3_gen1_dep16_batch32_eplb0_mtp0_615 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 6 + workers: 3 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 8 + + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 16 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['615'] + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp3-c666.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp3-c666.yaml new file mode 100644 index 0000000000..e932f2ba40 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp3-c666.yaml @@ -0,0 +1,138 @@ +schema: 2 +name: ctx3_gen1_dep8_batch64_eplb0_mtp3_666 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 6 + workers: 3 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 8 + + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 64 + max_num_tokens: 256 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 100 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['666'] + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b16-eplb0-mtp0-c666.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b16-eplb0-mtp0-c666.yaml new file mode 100644 index 0000000000..c4488a073d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b16-eplb0-mtp0-c666.yaml @@ -0,0 +1,126 @@ +schema: 2 +name: ctx4_gen1_dep32_batch16_eplb0_mtp0_666 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 8 + workers: 4 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 8 + + + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 32 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['666'] + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b8-eplb0-mtp3-c333.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b8-eplb0-mtp3-c333.yaml new file mode 100644 index 0000000000..96f0c77bf2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b8-eplb0-mtp3-c333.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: ctx4_gen1_dep32_batch8_eplb0_mtp3_333 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 8 + workers: 4 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 8 + + + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 100 + tensor_parallel_size: 32 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['333'] + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b32-eplb0-mtp3-c666.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b32-eplb0-mtp3-c666.yaml new file mode 100644 index 0000000000..fbcf7e9f4d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b32-eplb0-mtp3-c666.yaml @@ -0,0 +1,134 @@ +schema: 2 +name: ctx5_gen1_dep16_batch32_eplb0_mtp3_666 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 10 + workers: 5 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 8 + + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 100 + tensor_parallel_size: 16 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['666'] + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b64-eplb0-mtp0-c1229.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b64-eplb0-mtp0-c1229.yaml new file mode 100644 index 0000000000..8a605d7745 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b64-eplb0-mtp0-c1229.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: ctx5_gen1_dep16_batch64_eplb0_mtp0_1229 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 10 + workers: 5 + gpus: 8 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.4 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 8 + + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 64 + max_num_tokens: 64 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 16 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['1229'] + req_rate: "inf" + +frontend: + type: "dynamo" + nginx_container: "nginx-sqsh" + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep16-b32-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep16-b32-eplb0-mtp3.yaml new file mode 100644 index 0000000000..5b44c7c05f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep16-b32-eplb0-mtp3.yaml @@ -0,0 +1,134 @@ +schema: 2 +name: "ctx10_gen1_dep16_batch32_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 5 + workers: 10 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 4 + + workers: 1 + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "666" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp1.yaml new file mode 100644 index 0000000000..600abc192c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp1.yaml @@ -0,0 +1,162 @@ +schema: 2 +name: "ctx10_gen1_dep8_batch256_eplb0_mtp1" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 5 + workers: 10 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + + decode: + nodes: 2 + + workers: 1 + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 256 + max_num_tokens: 512 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2253" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-11p3d-dep4-b256-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-11p3d-dep4-b256-eplb0-mtp0.yaml new file mode 100644 index 0000000000..a010b091cc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-11p3d-dep4-b256-eplb0-mtp0.yaml @@ -0,0 +1,157 @@ +schema: 2 +name: "ctx11_gen3_dep4_batch256_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 6 + workers: 11 + gpus: 2 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + + decode: + nodes: 3 + + workers: 3 + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 256 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "3228" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-13p1d-dep16-b64-eplb256-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-13p1d-dep16-b64-eplb256-mtp3.yaml new file mode 100644 index 0000000000..2c5ed0f516 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-13p1d-dep16-b64-eplb256-mtp3.yaml @@ -0,0 +1,142 @@ +schema: 2 +name: "ctx13_gen1_dep16_batch64_eplb256_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 7 + workers: 13 + gpus: 2 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 4 + + workers: 1 + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 64 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + load_balancer: + num_slots: 256 + layer_updates_per_iter: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1127" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-14p1d-dep16-b128-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-14p1d-dep16-b128-eplb0-mtp0.yaml new file mode 100644 index 0000000000..22306cad30 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-14p1d-dep16-b128-eplb0-mtp0.yaml @@ -0,0 +1,140 @@ +schema: 2 +name: "ctx14_gen1_dep16_batch128_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 7 + workers: 14 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + + decode: + nodes: 4 + + workers: 1 + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 128 + max_num_tokens: 128 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2253" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0.yaml new file mode 100644 index 0000000000..dadf730a33 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0.yaml @@ -0,0 +1,128 @@ +schema: 2 +name: "ctx1_gen3_tep8_batch16_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + + decode: + nodes: 6 + + workers: 3 + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "72" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b8-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b8-eplb0-mtp3.yaml new file mode 100644 index 0000000000..88093b0878 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b8-eplb0-mtp3.yaml @@ -0,0 +1,133 @@ +schema: 2 +name: "ctx1_gen3_tep8_batch8_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 6 + + workers: 3 + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "33" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml new file mode 100644 index 0000000000..08eab29c53 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml @@ -0,0 +1,124 @@ +schema: 2 +name: "ctx1_gen4_tep8_batch1_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + + decode: + nodes: 8 + + workers: 4 + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "5" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3.yaml new file mode 100644 index 0000000000..6f6da3ebec --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3.yaml @@ -0,0 +1,130 @@ +schema: 2 +name: "ctx1_gen4_tep8_batch1_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 8 + + workers: 4 + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "5" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b2-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b2-eplb0-mtp0.yaml new file mode 100644 index 0000000000..8e9576773e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b2-eplb0-mtp0.yaml @@ -0,0 +1,125 @@ +schema: 2 +name: "ctx1_gen4_tep8_batch2_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + + decode: + nodes: 8 + + workers: 4 + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 2 + max_num_tokens: 2 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "12" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3.yaml new file mode 100644 index 0000000000..0f249c08bc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3.yaml @@ -0,0 +1,133 @@ +schema: 2 +name: "ctx1_gen4_tep8_batch4_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 8 + + workers: 4 + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 3 + - 4 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + allreduce_strategy: MNNVL + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "12x24" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p5d-tep4-b4-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p5d-tep4-b4-eplb0-mtp0.yaml new file mode 100644 index 0000000000..d139eeac50 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p5d-tep4-b4-eplb0-mtp0.yaml @@ -0,0 +1,125 @@ +schema: 2 +name: "ctx1_gen5_tep4_batch4_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + + decode: + nodes: 5 + + workers: 5 + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 4 + max_num_tokens: 4 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "5x15x30" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep32-b4-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep32-b4-eplb0-mtp3.yaml new file mode 100644 index 0000000000..cc902e3ba1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep32-b4-eplb0-mtp3.yaml @@ -0,0 +1,130 @@ +schema: 2 +name: "ctx4_gen1_dep32_batch4_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 4 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 8 + + workers: 1 + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "180" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep32-b16-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep32-b16-eplb0-mtp0.yaml new file mode 100644 index 0000000000..0ec09d8bc7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep32-b16-eplb0-mtp0.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: "ctx7_gen1_dep32_batch16_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 4 + workers: 7 + gpus: 2 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + + decode: + nodes: 8 + + workers: 1 + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "666" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-8p1d-dep32-b8-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-8p1d-dep32-b8-eplb0-mtp3.yaml new file mode 100644 index 0000000000..93f897ecb5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-8p1d-dep32-b8-eplb0-mtp3.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: "ctx8_gen1_dep32_batch8_eplb0_mtp3" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 4 + workers: 8 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + + decode: + nodes: 8 + + workers: 1 + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "308" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep16-b64-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep16-b64-eplb0-mtp0.yaml new file mode 100644 index 0000000000..1eee30498c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep16-b64-eplb0-mtp0.yaml @@ -0,0 +1,133 @@ +schema: 2 +name: "ctx9_gen1_dep16_batch64_eplb0_mtp0" + +model: + path: "dsr1" + container: "dynamo-trtllm" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 5 + workers: 9 + gpus: 2 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + tensor_parallel_size: 2 + moe_expert_parallel_size: 2 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + moe_config: + backend: TRTLLM + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + + decode: + nodes: 4 + + workers: 1 + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_ENABLE_PDL: "1" + + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 64 + max_num_tokens: 64 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + dtype: fp8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + cache_transceiver_config: + max_tokens_in_buffer: 16384 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1229" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-10p1d-dep16-b64-eplb0-mtp1-c1229.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-10p1d-dep16-b64-eplb0-mtp1-c1229.yaml new file mode 100644 index 0000000000..f39e6ec2f9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-10p1d-dep16-b64-eplb0-mtp1-c1229.yaml @@ -0,0 +1,141 @@ +schema: 2 +name: ctx10_gen1_dep16_batch64_eplb0_mtp1_1229 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 10 + workers: 10 + gpus: 4 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.1 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + tensor_parallel_size: 4 + + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 64 + max_num_tokens: 128 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + stream_interval: 100 + tensor_parallel_size: 16 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['1229'] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0-c4.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0-c4.yaml new file mode 100644 index 0000000000..0ab2f557eb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0-c4.yaml @@ -0,0 +1,128 @@ +schema: 2 +name: ctx1_gen4_tep8_batch1_eplb0_mtp0_4 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.1 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + + + decode: + nodes: 8 + workers: 4 + gpus: 8 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + allreduce_strategy: MNNVL + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['4'] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3-c8.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3-c8.yaml new file mode 100644 index 0000000000..55c20dd5de --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3-c8.yaml @@ -0,0 +1,134 @@ +schema: 2 +name: ctx1_gen4_tep8_batch1_eplb0_mtp3_8 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.1 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 4 + + + decode: + nodes: 8 + workers: 4 + gpus: 8 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + allreduce_strategy: MNNVL + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 100 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['8'] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml new file mode 100644 index 0000000000..cee5ba0fcc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml @@ -0,0 +1,128 @@ +schema: 2 +name: ctx1_gen4_tep8_batch4_eplb0_mtp0_24 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.1 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + + + decode: + nodes: 8 + workers: 4 + gpus: 8 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + allreduce_strategy: MNNVL + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 4 + max_num_tokens: 4 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['24'] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3-c24.yaml new file mode 100644 index 0000000000..9b6dfb6235 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3-c24.yaml @@ -0,0 +1,134 @@ +schema: 2 +name: ctx1_gen4_tep8_batch4_eplb0_mtp3_24 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.1 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 4 + + + decode: + nodes: 8 + workers: 4 + gpus: 8 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + allreduce_strategy: MNNVL + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 100 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['24'] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp0-c36.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp0-c36.yaml new file mode 100644 index 0000000000..bf912e02ea --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp0-c36.yaml @@ -0,0 +1,129 @@ +schema: 2 +name: ctx1_gen4_tep8_batch8_eplb0_mtp0_36 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.1 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + + + decode: + nodes: 8 + workers: 4 + gpus: 8 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + allreduce_strategy: MNNVL + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 8 + max_num_tokens: 8 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['36'] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-4p1d-dep16-b32-eplb0-mtp0-c666.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-4p1d-dep16-b32-eplb0-mtp0-c666.yaml new file mode 100644 index 0000000000..803171e831 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-4p1d-dep16-b32-eplb0-mtp0-c666.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: ctx4_gen1_dep16_batch32_eplb0_mtp0_666 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 4 + workers: 4 + gpus: 4 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.1 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 16 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['666'] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b16-eplb0-mtp0-c512.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b16-eplb0-mtp0-c512.yaml new file mode 100644 index 0000000000..2736b28f56 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b16-eplb0-mtp0-c512.yaml @@ -0,0 +1,129 @@ +schema: 2 +name: ctx6_gen1_dep32_batch16_eplb0_mtp0_512 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 6 + workers: 6 + gpus: 4 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.1 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + + + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 32 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['512'] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b8-eplb0-mtp3-c333.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b8-eplb0-mtp3-c333.yaml new file mode 100644 index 0000000000..52974739c3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b8-eplb0-mtp3-c333.yaml @@ -0,0 +1,134 @@ +schema: 2 +name: ctx6_gen1_dep32_batch8_eplb0_mtp3_333 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 6 + workers: 6 + gpus: 4 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.1 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 4 + + + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 100 + tensor_parallel_size: 32 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['333'] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep16-b64-eplb0-mtp0-c1229.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep16-b64-eplb0-mtp0-c1229.yaml new file mode 100644 index 0000000000..5acd5089d1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep16-b64-eplb0-mtp0-c1229.yaml @@ -0,0 +1,135 @@ +schema: 2 +name: ctx7_gen1_dep16_batch64_eplb0_mtp0_1229 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 7 + workers: 7 + gpus: 4 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.1 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 64 + max_num_tokens: 64 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 16 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['1229'] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b128-eplb0-mtp1-c1229.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b128-eplb0-mtp1-c1229.yaml new file mode 100644 index 0000000000..4c449203b0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b128-eplb0-mtp1-c1229.yaml @@ -0,0 +1,149 @@ +schema: 2 +name: ctx7_gen1_dep8_batch128_eplb0_mtp1_1229 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 7 + workers: 7 + gpus: 4 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8192 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.1 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + tensor_parallel_size: 4 + + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8192 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 128 + max_num_tokens: 256 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + stream_interval: 100 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['1229'] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b256-eplb0-mtp0-c2151.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b256-eplb0-mtp0-c2151.yaml new file mode 100644 index 0000000000..587ba16789 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b256-eplb0-mtp0-c2151.yaml @@ -0,0 +1,159 @@ +schema: 2 +name: ctx7_gen1_dep8_batch256_eplb0_mtp0_2151 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 7 + workers: 7 + gpus: 4 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.1 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 256 + max_num_tokens: 256 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['2151'] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-8p1d-dep16-b32-eplb0-mtp3-c666.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-8p1d-dep16-b32-eplb0-mtp3-c666.yaml new file mode 100644 index 0000000000..a1d7b254c3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-8p1d-dep16-b32-eplb0-mtp3-c666.yaml @@ -0,0 +1,137 @@ +schema: 2 +name: ctx8_gen1_dep16_batch32_eplb0_mtp3_666 + +model: + path: "dsr1-fp8" + container: "dynamo-trtllm" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 8 + workers: 8 + gpus: 4 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.1 + max_batch_size: 2 + max_num_tokens: 16384 + max_seq_len: 8232 + moe_config: + backend: DEEPGEMM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 4 + + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + TLLM_OVERRIDE_LAYER_NUM: "61" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + ENROOT_ALLOW_DEV: "yes" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TRTLLM_FORCE_COMM_METHOD: "NVLINK_TWO_SIDED" + ENABLE_CONFIGURABLE_MOE: "1" + + args: + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 + moe_config: + backend: DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + stream_interval: 100 + tensor_parallel_size: 16 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: ['666'] + req_rate: "inf" + +frontend: + type: "dynamo" + + enable_multiple_frontends: false + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +dynamo: + install: false + # The unversioned container alias does not identify a Dynamo version or hash. + # Use NATS for a recipe with an unverified Dynamo revision. + request_plane: "nats" +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p1d-dep16-b4-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p1d-dep16-b4-eplb0-mtp3.yaml new file mode 100644 index 0000000000..8b8846c8b0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p1d-dep16-b4-eplb0-mtp3.yaml @@ -0,0 +1,106 @@ +schema: 2 +name: h100_8k1k_ctx1dep16_gen1dep16_batch4_eplb0_mtp3 +model: + path: DeepSeek-R1-0528 + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post3" + precision: fp8 +resources: + gpu_type: h100 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 1 + env: + UCX_CUDA_IPC_ENABLE_MNNVL: n + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP: '1' + TRTLLM_FORCE_ALLTOALL_METHOD: DeepEP + args: + max_batch_size: 1 + max_num_tokens: 8224 + max_seq_len: 8232 + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + enable_chunked_prefill: true + moe_config: + backend: WIDEEP + max_num_tokens: 16384 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.3 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8256 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 2 + workers: 1 + env: + NCCL_NVLS_ENABLE: '0' + UCX_CUDA_IPC_ENABLE_MNNVL: n + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP: '1' + TRTLLM_FORCE_ALLTOALL_METHOD: DeepEP + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 4 + max_num_tokens: 128 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: WIDEEP + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8256 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '77' + req_rate: inf +frontend: + type: dynamo + enable_multiple_frontends: false +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p2d-tep16-b32-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p2d-tep16-b32-eplb0-mtp3.yaml new file mode 100644 index 0000000000..51cdf9c090 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p2d-tep16-b32-eplb0-mtp3.yaml @@ -0,0 +1,108 @@ +schema: 2 +name: h100_8k1k_ctx1dep16_gen2tep16_batch32_eplb0_mtp3 +model: + path: DeepSeek-R1-0528 + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post3" + precision: fp8 +resources: + gpu_type: h100 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 1 + env: + UCX_CUDA_IPC_ENABLE_MNNVL: n + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP: '1' + TRTLLM_FORCE_ALLTOALL_METHOD: DeepEP + args: + max_batch_size: 1 + max_num_tokens: 8224 + max_seq_len: 8232 + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + enable_chunked_prefill: false + moe_config: + backend: WIDEEP + max_num_tokens: 16384 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.3 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8256 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 4 + workers: 2 + env: + NCCL_NVLS_ENABLE: '0' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + UCX_CUDA_IPC_ENABLE_MNNVL: n + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 32 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8256 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '78' + req_rate: inf +frontend: + type: dynamo + enable_multiple_frontends: false +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p2d-tep16-b64-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p2d-tep16-b64-eplb0-mtp0.yaml new file mode 100644 index 0000000000..8a06e91682 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p2d-tep16-b64-eplb0-mtp0.yaml @@ -0,0 +1,107 @@ + +schema: 2 +name: "h100_8k1k_ctx1dep16_gen2tep16_batch64_eplb0_mtp0" + +model: + path: "DeepSeek-R1-0528" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post3" + precision: "fp8" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 1 + env: + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP: "1" + TRTLLM_FORCE_ALLTOALL_METHOD: "DeepEP" + + args: + max_batch_size: 1 + max_num_tokens: 8224 + max_seq_len: 8232 + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + enable_chunked_prefill: false + moe_config: + backend: WIDEEP + max_num_tokens: 16384 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.3 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8256 + + + decode: + nodes: 4 + workers: 2 + env: + NCCL_NVLS_ENABLE: "0" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TLLM_LOG_LEVEL: "INFO" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 64 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64] + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8256 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "154" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # There are errors about colliding on port 8080, and others. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b1-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b1-eplb0-mtp0.yaml new file mode 100644 index 0000000000..7e27a3b932 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b1-eplb0-mtp0.yaml @@ -0,0 +1,99 @@ +schema: 2 +name: h100_8k1k_ctx1dep16_gen3tep16_batch1_eplb0_mtp0 +model: + path: DeepSeek-R1-0528 + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post3" + precision: fp8 +resources: + gpu_type: h100 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 1 + env: + UCX_CUDA_IPC_ENABLE_MNNVL: n + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP: '1' + TRTLLM_FORCE_ALLTOALL_METHOD: DeepEP + args: + max_batch_size: 1 + max_num_tokens: 8224 + max_seq_len: 8232 + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + enable_chunked_prefill: false + moe_config: + backend: WIDEEP + max_num_tokens: 16384 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.3 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8256 + decode: + nodes: 6 + workers: 3 + env: + NCCL_NVLS_ENABLE: '0' + UCX_CUDA_IPC_ENABLE_MNNVL: n + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP: '1' + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 1 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8256 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '6' + req_rate: inf +frontend: + type: dynamo + enable_multiple_frontends: false +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b1-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b1-eplb0-mtp3.yaml new file mode 100644 index 0000000000..1f3fd709d9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b1-eplb0-mtp3.yaml @@ -0,0 +1,104 @@ +schema: 2 +name: h100_8k1k_ctx1dep16_gen3tep16_batch1_eplb0_mtp3 +model: + path: DeepSeek-R1-0528 + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post3" + precision: fp8 +resources: + gpu_type: h100 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 1 + env: + UCX_CUDA_IPC_ENABLE_MNNVL: n + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP: '1' + TRTLLM_FORCE_ALLTOALL_METHOD: DeepEP + args: + max_batch_size: 1 + max_num_tokens: 8224 + max_seq_len: 8232 + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + enable_chunked_prefill: true + moe_config: + backend: WIDEEP + max_num_tokens: 16384 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.3 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8256 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 6 + workers: 3 + env: + NCCL_NVLS_ENABLE: '0' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + UCX_CUDA_IPC_ENABLE_MNNVL: n + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 1 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8256 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '6' + req_rate: inf +frontend: + type: dynamo + enable_multiple_frontends: false +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b2-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b2-eplb0-mtp0.yaml new file mode 100644 index 0000000000..740de40de0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b2-eplb0-mtp0.yaml @@ -0,0 +1,107 @@ + +schema: 2 +name: "h100_8k1k_ctx1dep16_gen3tep16_batch2_eplb0_mtp0" + +model: + path: "DeepSeek-R1-0528" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post3" + precision: "fp8" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 1 + env: + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP: "1" + TRTLLM_FORCE_ALLTOALL_METHOD: "DeepEP" + + args: + max_batch_size: 1 + max_num_tokens: 8224 + max_seq_len: 8232 + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + enable_chunked_prefill: false + moe_config: + backend: WIDEEP + max_num_tokens: 16384 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.3 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8256 + + + decode: + nodes: 6 + workers: 3 + env: + NCCL_NVLS_ENABLE: "0" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TLLM_LOG_LEVEL: "INFO" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 2 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4] + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8256 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "9" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # There are errors about colliding on port 8080, and others. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b2-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b2-eplb0-mtp3.yaml new file mode 100644 index 0000000000..03ac6a374b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b2-eplb0-mtp3.yaml @@ -0,0 +1,104 @@ +schema: 2 +name: h100_8k1k_ctx1dep16_gen3tep16_batch2_eplb0_mtp3 +model: + path: DeepSeek-R1-0528 + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post3" + precision: fp8 +resources: + gpu_type: h100 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 1 + env: + UCX_CUDA_IPC_ENABLE_MNNVL: n + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP: '1' + TRTLLM_FORCE_ALLTOALL_METHOD: DeepEP + args: + max_batch_size: 1 + max_num_tokens: 8224 + max_seq_len: 8232 + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + enable_chunked_prefill: true + moe_config: + backend: WIDEEP + max_num_tokens: 16384 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.3 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8256 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 6 + workers: 3 + env: + NCCL_NVLS_ENABLE: '0' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + UCX_CUDA_IPC_ENABLE_MNNVL: n + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 2 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8256 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '9' + req_rate: inf +frontend: + type: dynamo + enable_multiple_frontends: false +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b8-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b8-eplb0-mtp0.yaml new file mode 100644 index 0000000000..b22aa1922c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b8-eplb0-mtp0.yaml @@ -0,0 +1,107 @@ + +schema: 2 +name: "h100_8k1k_ctx1dep16_gen3tep16_batch8_eplb0_mtp0" + +model: + path: "DeepSeek-R1-0528" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post3" + precision: "fp8" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 1 + env: + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP: "1" + TRTLLM_FORCE_ALLTOALL_METHOD: "DeepEP" + + args: + max_batch_size: 1 + max_num_tokens: 8224 + max_seq_len: 8232 + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + enable_chunked_prefill: true + moe_config: + backend: WIDEEP + max_num_tokens: 16384 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.3 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8256 + + + decode: + nodes: 6 + workers: 3 + env: + NCCL_NVLS_ENABLE: "0" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + TLLM_LOG_LEVEL: "INFO" + UCX_CUDA_IPC_ENABLE_MNNVL: "n" + + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 8 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8] + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8256 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "30" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # There are errors about colliding on port 8080, and others. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b8-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b8-eplb0-mtp3.yaml new file mode 100644 index 0000000000..4e1eb0d921 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b8-eplb0-mtp3.yaml @@ -0,0 +1,105 @@ +schema: 2 +name: h100_8k1k_ctx1dep16_gen3tep16_batch8_eplb0_mtp3 +model: + path: DeepSeek-R1-0528 + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post3" + precision: fp8 +resources: + gpu_type: h100 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 1 + env: + UCX_CUDA_IPC_ENABLE_MNNVL: n + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP: '1' + TRTLLM_FORCE_ALLTOALL_METHOD: DeepEP + args: + max_batch_size: 1 + max_num_tokens: 8224 + max_seq_len: 8232 + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + enable_chunked_prefill: false + moe_config: + backend: WIDEEP + max_num_tokens: 16384 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.3 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8256 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 6 + workers: 3 + env: + NCCL_NVLS_ENABLE: '0' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + UCX_CUDA_IPC_ENABLE_MNNVL: n + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 8 + max_num_tokens: 256 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8256 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '30' + req_rate: inf +frontend: + type: dynamo + enable_multiple_frontends: false +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-2p1d-dep16-b16-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-2p1d-dep16-b16-eplb0-mtp0.yaml new file mode 100644 index 0000000000..91dce0d59c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-2p1d-dep16-b16-eplb0-mtp0.yaml @@ -0,0 +1,102 @@ +schema: 2 +name: h100_8k1k_ctx2dep16_gen1dep16_batch16_eplb0_mtp0 +model: + path: DeepSeek-R1-0528 + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post3" + precision: fp8 +resources: + gpu_type: h100 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 4 + workers: 2 + env: + UCX_CUDA_IPC_ENABLE_MNNVL: n + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP: '1' + TRTLLM_FORCE_ALLTOALL_METHOD: DeepEP + args: + max_batch_size: 1 + max_num_tokens: 8224 + max_seq_len: 8232 + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + enable_chunked_prefill: false + moe_config: + backend: WIDEEP + max_num_tokens: 16384 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.3 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8256 + decode: + nodes: 2 + workers: 1 + env: + NCCL_NVLS_ENABLE: '0' + UCX_CUDA_IPC_ENABLE_MNNVL: n + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP: '1' + TRTLLM_FORCE_ALLTOALL_METHOD: DeepEP + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + pipeline_parallel_size: 1 + max_batch_size: 16 + max_num_tokens: 128 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: WIDEEP + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8256 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '308' + req_rate: inf +frontend: + type: dynamo + enable_multiple_frontends: false +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-2p1d-dep16-b8-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-2p1d-dep16-b8-eplb0-mtp3.yaml new file mode 100644 index 0000000000..5eb9fdc23e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-2p1d-dep16-b8-eplb0-mtp3.yaml @@ -0,0 +1,107 @@ +schema: 2 +name: h100_8k1k_ctx2dep16_gen1dep16_batch8_eplb0_mtp3 +model: + path: DeepSeek-R1-0528 + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post3" + precision: fp8 +resources: + gpu_type: h100 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 4 + workers: 2 + env: + UCX_CUDA_IPC_ENABLE_MNNVL: n + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP: '1' + TRTLLM_FORCE_ALLTOALL_METHOD: DeepEP + args: + max_batch_size: 1 + max_num_tokens: 8224 + max_seq_len: 8232 + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + pipeline_parallel_size: 1 + print_iter_log: true + cuda_graph_config: + disable_overlap_scheduler: true + enable_chunked_prefill: true + moe_config: + backend: WIDEEP + max_num_tokens: 16384 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.3 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 8256 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 2 + workers: 1 + env: + NCCL_NVLS_ENABLE: '0' + UCX_CUDA_IPC_ENABLE_MNNVL: n + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP: '1' + TRTLLM_FORCE_ALLTOALL_METHOD: DeepEP + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 8 + max_num_tokens: 128 + max_seq_len: 9256 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + print_iter_log: true + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + dtype: fp8 + moe_config: + backend: WIDEEP + use_low_precision_moe_combine: true + cache_transceiver_config: + max_tokens_in_buffer: 8256 + backend: UCX + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '154' + req_rate: inf +frontend: + type: dynamo + enable_multiple_frontends: false +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p1d-dep8-b256-eplb0-mtp0-c128.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p1d-dep8-b256-eplb0-mtp0-c128.yaml new file mode 100644 index 0000000000..6d29cf84b7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p1d-dep8-b256-eplb0-mtp0-c128.yaml @@ -0,0 +1,118 @@ +schema: 2 +name: "c128_ctx1_gen1_dep8_batch256_eplb0_mtp0" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (DEP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + # Matches E2E standalone ctx_config.yaml + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + + decode: + nodes: 1 + + workers: 1 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (DEP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + # Matches E2E standalone gen_config.yaml (DEP c=128) + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_chunked_prefill: false + max_batch_size: 256 + max_num_tokens: 256 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128, 256] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "128" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p1d-dep8-b32-eplb0-mtp2-c64.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p1d-dep8-b32-eplb0-mtp2-c64.yaml new file mode 100644 index 0000000000..678260d324 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p1d-dep8-b32-eplb0-mtp2-c64.yaml @@ -0,0 +1,121 @@ +schema: 2 +name: "c64_ctx1_gen1_dep8_batch32_eplb0_mtp2" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (MTP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 2 + decode: + nodes: 1 + + workers: 1 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (MTP c=64) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_chunked_prefill: false + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 32] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 2 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "64" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p3d-tep8-b32-eplb0-mtp0-c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p3d-tep8-b32-eplb0-mtp0-c48.yaml new file mode 100644 index 0000000000..0ffaa88578 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p3d-tep8-b32-eplb0-mtp0-c48.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: "c16_ctx1_gen3_tep8_batch32_eplb0_mtp0" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (TEP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + decode: + nodes: 3 + + workers: 3 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (TEP c=16) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 32] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "48" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p3d-tep8-b32-eplb0-mtp2-c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p3d-tep8-b32-eplb0-mtp2-c48.yaml new file mode 100644 index 0000000000..159a19aa6a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p3d-tep8-b32-eplb0-mtp2-c48.yaml @@ -0,0 +1,121 @@ +schema: 2 +name: "c16_ctx1_gen3_tep8_batch32_eplb0_mtp2" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (MTP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 2 + decode: + nodes: 3 + + workers: 3 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (MTP c=16) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 32] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 2 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "48" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p6d-tep8-b16-eplb0-mtp0-c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p6d-tep8-b16-eplb0-mtp0-c48.yaml new file mode 100644 index 0000000000..26b57c7336 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p6d-tep8-b16-eplb0-mtp0-c48.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: "c8_ctx1_gen6_tep8_batch16_eplb0_mtp0" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (TEP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + decode: + nodes: 6 + + workers: 6 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (TEP c=8) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "48" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p6d-tep8-b32-eplb0-mtp3-c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p6d-tep8-b32-eplb0-mtp3-c48.yaml new file mode 100644 index 0000000000..68c9cecf6b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p6d-tep8-b32-eplb0-mtp3-c48.yaml @@ -0,0 +1,121 @@ +schema: 2 +name: "c8_ctx1_gen6_tep8_batch32_eplb0_mtp3" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (MTP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 6 + + workers: 6 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (MTP c=8) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 32] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "48" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b1-eplb0-mtp0-c9.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b1-eplb0-mtp0-c9.yaml new file mode 100644 index 0000000000..f4acc636d5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b1-eplb0-mtp0-c9.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: "c1_ctx1_gen7_tep8_batch1_eplb0_mtp0" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (TEP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + decode: + nodes: 7 + + workers: 7 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (TEP c=4) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "9" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b1-eplb0-mtp3-c9.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b1-eplb0-mtp3-c9.yaml new file mode 100644 index 0000000000..bc547bb206 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b1-eplb0-mtp3-c9.yaml @@ -0,0 +1,121 @@ +schema: 2 +name: "c1_ctx1_gen7_tep8_batch1_eplb0_mtp3" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (MTP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 7 + + workers: 7 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (MTP c=4) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "9" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b32-eplb0-mtp0-c28.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b32-eplb0-mtp0-c28.yaml new file mode 100644 index 0000000000..a16cedac66 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b32-eplb0-mtp0-c28.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: "c4_ctx1_gen7_tep8_batch32_eplb0_mtp0" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (TEP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + decode: + nodes: 7 + + workers: 7 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (TEP c=4) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 32] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "28" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b32-eplb0-mtp3-c28.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b32-eplb0-mtp3-c28.yaml new file mode 100644 index 0000000000..58b602d0a0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b32-eplb0-mtp3-c28.yaml @@ -0,0 +1,121 @@ +schema: 2 +name: "c4_ctx1_gen7_tep8_batch32_eplb0_mtp3" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (MTP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 7 + + workers: 7 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (MTP c=4) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 32] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "28" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-2p1d-dep8-b32-eplb0-mtp2-c128.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-2p1d-dep8-b32-eplb0-mtp2-c128.yaml new file mode 100644 index 0000000000..ff2f750024 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-2p1d-dep8-b32-eplb0-mtp2-c128.yaml @@ -0,0 +1,121 @@ +schema: 2 +name: "c128_ctx2_gen1_dep8_batch32_eplb0_mtp2" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (MTP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 2 + decode: + nodes: 1 + + workers: 1 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (MTP c=128) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_chunked_prefill: false + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 32] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 2 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "128" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-2p3d-dep8-b128-eplb0-mtp0-c192.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-2p3d-dep8-b128-eplb0-mtp0-c192.yaml new file mode 100644 index 0000000000..d84d12d166 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-2p3d-dep8-b128-eplb0-mtp0-c192.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: "c64_ctx2_gen3_dep8_batch128_eplb0_mtp0" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (DEP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + decode: + nodes: 3 + + workers: 3 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (DEP c=64) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_chunked_prefill: false + max_batch_size: 128 + max_num_tokens: 128 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "192" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-2p5d-tep8-b128-eplb0-mtp0-c160.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-2p5d-tep8-b128-eplb0-mtp0-c160.yaml new file mode 100644 index 0000000000..69ff3f63b3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-2p5d-tep8-b128-eplb0-mtp0-c160.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: "c32_ctx2_gen5_tep8_batch128_eplb0_mtp0" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (TEP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + decode: + nodes: 5 + + workers: 5 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (TEP c=32) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 128 + max_num_tokens: 128 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "160" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p1d-dep8-b32-eplb0-mtp2-c256.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p1d-dep8-b32-eplb0-mtp2-c256.yaml new file mode 100644 index 0000000000..a984eaa2c8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p1d-dep8-b32-eplb0-mtp2-c256.yaml @@ -0,0 +1,121 @@ +schema: 2 +name: "c256_ctx3_gen1_dep8_batch32_eplb0_mtp2" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 3 + workers: 3 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (MTP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 2 + decode: + nodes: 1 + + workers: 1 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (MTP c=256) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_chunked_prefill: false + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 32] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 2 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "256" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p1d-dep8-b512-eplb0-mtp0-c512.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p1d-dep8-b512-eplb0-mtp0-c512.yaml new file mode 100644 index 0000000000..750fec9cac --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p1d-dep8-b512-eplb0-mtp0-c512.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: "c512_ctx3_gen1_dep8_batch512_eplb0_mtp0" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 3 + workers: 3 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (DEP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + decode: + nodes: 1 + + workers: 1 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (DEP c=512) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_chunked_prefill: false + max_batch_size: 512 + max_num_tokens: 512 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp1-c512.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp1-c512.yaml new file mode 100644 index 0000000000..8ccf739fe4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp1-c512.yaml @@ -0,0 +1,121 @@ +schema: 2 +name: "c512_ctx3_gen1_dep8_batch64_eplb0_mtp1" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 3 + workers: 3 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (MTP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + decode: + nodes: 1 + + workers: 1 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (MTP c=512) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_chunked_prefill: false + max_batch_size: 64 + max_num_tokens: 256 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 32, 64] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p5d-tep8-b32-eplb0-mtp3-c160.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p5d-tep8-b32-eplb0-mtp3-c160.yaml new file mode 100644 index 0000000000..d9063d91f7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p5d-tep8-b32-eplb0-mtp3-c160.yaml @@ -0,0 +1,121 @@ +schema: 2 +name: "c32_ctx3_gen5_tep8_batch32_eplb0_mtp3" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 3 + workers: 3 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (MTP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 5 + + workers: 5 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (MTP c=32) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 32] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "160" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-5p3d-dep8-b256-eplb0-mtp0-c768.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-5p3d-dep8-b256-eplb0-mtp0-c768.yaml new file mode 100644 index 0000000000..5e5d015dd8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-5p3d-dep8-b256-eplb0-mtp0-c768.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: "c256_ctx5_gen3_dep8_batch256_eplb0_mtp0" + +model: + path: "dsr1" + container: "nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1" + precision: "fp8" + +sbatch_directives: + cpus-per-gpu: "16" + +resources: + gpu_type: "h200" + gpus_per_node: 8 + +engine: trtllm +roles: + prefill: + nodes: 5 + workers: 5 + + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Prefill Worker Config for Dynamo DSR1 (DEP mode) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_chunked_prefill: false + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 32768 + moe_config: + backend: CUTLASS + cuda_graph_config: + disable_overlap_scheduler: true + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + decode: + nodes: 3 + + workers: 3 + env: + UCX_TLS: "rc,dc,ud,cuda_copy,cuda_ipc,gdr_copy,tcp" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + + args: + # Decode Worker Config for Dynamo DSR1 (DEP c=256) + # ISL/OSL: 8k/1k, TP=8 on H200 + backend: pytorch + trust_remote_code: true + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_chunked_prefill: false + max_batch_size: 256 + max_num_tokens: 256 + max_seq_len: 9256 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + dtype: fp8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + moe_config: + backend: CUTLASS + use_low_precision_moe_combine: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128, 256] + disable_overlap_scheduler: false + print_iter_log: true + # Performance tuning + stream_interval: 100 + num_postprocess_workers: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "768" + req_rate: "inf" + +frontend: + type: "dynamo" + enable_multiple_frontends: false # For some reason, the H200 cluster doesn't like nginx. + +dynamo: + install: false + # The container tag identifies a pre-39d2a68 Dynamo release; the image is not digest-pinned. + # Use NATS for a recipe prior to Dynamo commit 39d2a68. + request_plane: "nats" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-dep8-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-dep8-dep8-stp.yaml new file mode 100644 index 0000000000..dd889367b7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-dep8-dep8-stp.yaml @@ -0,0 +1,154 @@ +schema: 2 +name: "disagg-b200-1p1d-dep8-dep8-c512" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 +health_check: + max_attempts: 240 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_OPT_FP8_WO_A_GEMM: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 65536 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + load-balance-method: "total_requests" + moe-a2a-backend: "megamoe" + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.056 + context-length: 9216 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 21504 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "256x512" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-dep8-tp8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-dep8-tp8-stp.yaml new file mode 100644 index 0000000000..85403a7c5b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-dep8-tp8-stp.yaml @@ -0,0 +1,146 @@ +schema: 2 +name: "disagg-b200-1p1d-dep8-tp8-4-c64" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 +health_check: + max_attempts: 240 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_OPT_FP8_WO_A_GEMM: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 65536 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "32x64" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp8-d-tp8-b8-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp8-d-tp8-b8-c1-mtp.yaml new file mode 100644 index 0000000000..bfcaedde9c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp8-d-tp8-b8-c1-mtp.yaml @@ -0,0 +1,133 @@ +schema: 2 +name: "disagg-b200-8k1k-low-latency-1p1d-tp8-tp8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 +health_check: + max_attempts: 240 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 16 + cuda-graph-max-bs: 8 + chunked-prefill-size: 65536 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp8-ep8-dp8-d-tp8-ep8-dp8-b1536-c256x1024-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp8-ep8-dp8-d-tp8-ep8-dp8-b1536-c256x1024-mtp.yaml new file mode 100644 index 0000000000..bbadd4484e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp8-ep8-dp8-d-tp8-ep8-dp8-b1536-c256x1024-mtp.yaml @@ -0,0 +1,141 @@ +schema: 2 +name: "disagg-b200-8k1k-mid-curve-1p1d-dep8-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 +health_check: + max_attempts: 240 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + mem-fraction-static: 0.8 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 65536 + stream-interval: 60 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + mem-fraction-static: 0.9 + max-running-requests: 1536 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "256x1024" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-tp8-tp8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-tp8-tp8-stp.yaml new file mode 100644 index 0000000000..0818a8d2d9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-tp8-tp8-stp.yaml @@ -0,0 +1,123 @@ +schema: 2 +name: "disagg-b200-1p1d-tp8-tp8-c1" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 +health_check: + max_attempts: 240 + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.90 + max-running-requests: 1024 + cuda-graph-max-bs: 512 + chunked-prefill-size: 65536 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p2d-dep8-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p2d-dep8-dep8-stp.yaml new file mode 100644 index 0000000000..1b4f2ddb4f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p2d-dep8-dep8-stp.yaml @@ -0,0 +1,155 @@ +schema: 2 +name: "disagg-b200-1p2d-dep8-dep8-c256" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 +health_check: + max_attempts: 240 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_OPT_FP8_WO_A_GEMM: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 65536 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + load-balance-method: "total_requests" + moe-a2a-backend: "megamoe" + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.056 + context-length: 9216 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 21504 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "256" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p2d-p-tp8-ep8-dp8-d-tp8-ep8-dp8-b1024-c256-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p2d-p-tp8-ep8-dp8-d-tp8-ep8-dp8-b1024-c256-mtp.yaml new file mode 100644 index 0000000000..fd8f5da2eb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p2d-p-tp8-ep8-dp8-d-tp8-ep8-dp8-b1024-c256-mtp.yaml @@ -0,0 +1,152 @@ +schema: 2 +name: "disagg-b200-8k1k-mid-curve-1p2d-dep8-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 +health_check: + max_attempts: 240 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.8 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 65536 + stream-interval: 60 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.85 + max-running-requests: 1024 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p4d-dep8-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p4d-dep8-dep8-stp.yaml new file mode 100644 index 0000000000..b86c71b368 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p4d-dep8-dep8-stp.yaml @@ -0,0 +1,155 @@ +schema: 2 +name: "disagg-b200-1p4d-dep8-dep8-c256" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 +health_check: + max_attempts: 240 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_OPT_FP8_WO_A_GEMM: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 65536 + + decode: + nodes: 4 + workers: 4 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + load-balance-method: "total_requests" + moe-a2a-backend: "megamoe" + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.056 + context-length: 9216 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 21504 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "256" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p4d-dep8-tp8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p4d-dep8-tp8-stp.yaml new file mode 100644 index 0000000000..f36e10f3dd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p4d-dep8-tp8-stp.yaml @@ -0,0 +1,146 @@ +schema: 2 +name: "disagg-b200-1p4d-dep8-tp8-c64" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 +health_check: + max_attempts: 240 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_OPT_FP8_WO_A_GEMM: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 65536 + + decode: + nodes: 4 + workers: 4 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "64" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p6d-p-tp8-ep8-dp8-d-tp8-b128-c32x64x128-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p6d-p-tp8-ep8-dp8-d-tp8-b128-c32x64x128-mtp.yaml new file mode 100644 index 0000000000..790b475104 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p6d-p-tp8-ep8-dp8-d-tp8-b128-c32x64x128-mtp.yaml @@ -0,0 +1,140 @@ +schema: 2 +name: "disagg-b200-8k1k-low-latency-1p6d-dep8-tp8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 +health_check: + max_attempts: 240 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.9 + max-running-requests: 256 + cuda-graph-max-bs: 128 + chunked-prefill-size: 65536 + + decode: + nodes: 6 + workers: 6 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "32x64x128" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-3p2d-dep8-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-3p2d-dep8-dep8-stp.yaml new file mode 100644 index 0000000000..bc1f6bd5f8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-3p2d-dep8-dep8-stp.yaml @@ -0,0 +1,155 @@ +schema: 2 +name: "disagg-b200-3p2d-dep8-dep8-c2048" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 +health_check: + max_attempts: 240 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 3 + workers: 3 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_OPT_FP8_WO_A_GEMM: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 65536 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + load-balance-method: "total_requests" + moe-a2a-backend: "megamoe" + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.056 + context-length: 9216 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 21504 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-4p2d-dep8-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-4p2d-dep8-dep8-stp.yaml new file mode 100644 index 0000000000..621c75b972 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-4p2d-dep8-dep8-stp.yaml @@ -0,0 +1,155 @@ +schema: 2 +name: "disagg-b200-4p2d-dep8-dep8-c4096" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 +health_check: + max_attempts: 240 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 4 + workers: 4 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_OPT_FP8_WO_A_GEMM: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 65536 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + load-balance-method: "total_requests" + moe-a2a-backend: "megamoe" + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.056 + context-length: 9216 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 21504 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-5p2d-dep8-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-5p2d-dep8-dep8-stp.yaml new file mode 100644 index 0000000000..7409b1bbde --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-5p2d-dep8-dep8-stp.yaml @@ -0,0 +1,155 @@ +schema: 2 +name: "disagg-b200-5p2d-dep8-dep8-c6144" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 +health_check: + max_attempts: 240 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 5 + workers: 5 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_OPT_FP8_WO_A_GEMM: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 65536 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + load-balance-method: "total_requests" + moe-a2a-backend: "megamoe" + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.056 + context-length: 9216 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 21504 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "6144" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-5p3d-p-tp8-ep8-dp8-d-tp8-ep8-dp8-b2560-c6144-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-5p3d-p-tp8-ep8-dp8-d-tp8-ep8-dp8-b2560-c6144-mtp.yaml new file mode 100644 index 0000000000..b5877dcc67 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-5p3d-p-tp8-ep8-dp8-d-tp8-ep8-dp8-b2560-c6144-mtp.yaml @@ -0,0 +1,141 @@ +schema: 2 +name: "disagg-b200-8k1k-mid-curve-5p3d-dep8-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 +health_check: + max_attempts: 240 + +engine: sglang +roles: + prefill: + nodes: 5 + workers: 5 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + mem-fraction-static: 0.8 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 65536 + stream-interval: 60 + + decode: + nodes: 3 + workers: 3 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + mem-fraction-static: 0.9 + max-running-requests: 2560 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "6144" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-6p2d-dep8-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-6p2d-dep8-dep8-stp.yaml new file mode 100644 index 0000000000..d47b594593 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/8k1k/disagg-6p2d-dep8-dep8-stp.yaml @@ -0,0 +1,155 @@ +schema: 2 +name: "disagg-b200-6p2d-dep8-dep8-c8192" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 +health_check: + max_attempts: 240 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 6 + workers: 6 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_OPT_FP8_WO_A_GEMM: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 65536 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + load-balance-method: "total_requests" + moe-a2a-backend: "megamoe" + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.056 + context-length: 9216 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 21504 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "8192" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p1d-dep4-dep8-mtp.yaml new file mode 100644 index 0000000000..420ed7ea8f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,147 @@ +schema: 2 +name: "disagg-b300-1p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "41882ae9b07232eed4850fb1daf8c958abb2556a" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + mem-fraction-static: 0.9 + max-running-requests: 256 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.94 + max-running-requests: 3072 + cuda-graph-max-bs: 256 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p1d-dep4-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p1d-dep4-dep8-stp.yaml new file mode 100644 index 0000000000..b2a214c375 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p1d-dep4-dep8-stp.yaml @@ -0,0 +1,158 @@ +schema: 2 +name: "disagg-b300-1p1d-dep4-dep8" + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "41882ae9b07232eed4850fb1daf8c958abb2556a" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + enable-dp-lm-head: true + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.20 + context-length: 9216 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 18432 + cuda-graph-max-bs: 1280 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "256" + req_rate: "inf" + use_chat_template: false + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p1d-tp4-tp4-mtp.yaml new file mode 100644 index 0000000000..a87a4aee06 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p1d-tp4-tp4-mtp.yaml @@ -0,0 +1,138 @@ +schema: 2 +name: "disagg-b300-1p1d-tp4-tp4-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "41882ae9b07232eed4850fb1daf8c958abb2556a" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + chunked-prefill-size: 32768 + + + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p1d-tp4-tp4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p1d-tp4-tp4-stp.yaml new file mode 100644 index 0000000000..bc5dc81ec5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p1d-tp4-tp4-stp.yaml @@ -0,0 +1,133 @@ +schema: 2 +name: "disagg-b300-1p1d-tp4-tp4" + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "41882ae9b07232eed4850fb1daf8c958abb2556a" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.90 + max-running-requests: 32 + cuda-graph-max-bs: 32 + chunked-prefill-size: 32768 + + + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 64 + cuda-graph-max-bs: 64 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1x8x16x32" + req_rate: "inf" + use_chat_template: false + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p2d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p2d-dep4-dep8-mtp.yaml new file mode 100644 index 0000000000..17b91bd7e4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p2d-dep4-dep8-mtp.yaml @@ -0,0 +1,148 @@ +schema: 2 +name: "disagg-b300-1p2d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "41882ae9b07232eed4850fb1daf8c958abb2556a" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 256 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.94 + max-running-requests: 3072 + cuda-graph-max-bs: 256 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p6d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p6d-dep4-tp4-mtp.yaml new file mode 100644 index 0000000000..7a75a7d59f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p6d-dep4-tp4-mtp.yaml @@ -0,0 +1,142 @@ +schema: 2 +name: "disagg-b300-1p6d-dep4-tp4-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "41882ae9b07232eed4850fb1daf8c958abb2556a" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + chunked-prefill-size: 32768 + + decode: + nodes: 3 + workers: 6 + gpus: 4 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "8x32x64" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-2p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-2p1d-dep4-dep8-mtp.yaml new file mode 100644 index 0000000000..bf98394f2e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-2p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,148 @@ +schema: 2 +name: "disagg-b300-2p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "41882ae9b07232eed4850fb1daf8c958abb2556a" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 3072 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "512" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-2p1d-dep4-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-2p1d-dep4-dep8-stp.yaml new file mode 100644 index 0000000000..27702b94f4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-2p1d-dep4-dep8-stp.yaml @@ -0,0 +1,158 @@ +schema: 2 +name: "disagg-b300-2p1d-dep4-dep8" + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "41882ae9b07232eed4850fb1daf8c958abb2556a" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + enable-dp-lm-head: true + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.20 + context-length: 9216 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 18432 + cuda-graph-max-bs: 1280 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "768" + req_rate: "inf" + use_chat_template: false + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-4p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-4p1d-dep4-dep8-mtp.yaml new file mode 100644 index 0000000000..a86ab417c7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-4p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,148 @@ +schema: 2 +name: "disagg-b300-4p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "41882ae9b07232eed4850fb1daf8c958abb2556a" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +engine: sglang +roles: + prefill: + nodes: 2 + workers: 4 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 3072 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1024" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-4p1d-dep4-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-4p1d-dep4-dep8-stp.yaml new file mode 100644 index 0000000000..bece74ac8a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-4p1d-dep4-dep8-stp.yaml @@ -0,0 +1,158 @@ +schema: 2 +name: "disagg-b300-4p1d-dep4-dep8" + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "41882ae9b07232eed4850fb1daf8c958abb2556a" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 4 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + enable-dp-lm-head: true + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.20 + context-length: 9216 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 18432 + cuda-graph-max-bs: 1280 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" + use_chat_template: false + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-6p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-6p1d-dep4-dep8-mtp.yaml new file mode 100644 index 0000000000..c5916adccd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-6p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,148 @@ +schema: 2 +name: "disagg-b300-6p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "41882ae9b07232eed4850fb1daf8c958abb2556a" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +engine: sglang +roles: + prefill: + nodes: 3 + workers: 6 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + + mem-fraction-static: 0.9 + max-running-requests: 4096 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 9216 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "2048" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-6p1d-dep4-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-6p1d-dep4-dep8-stp.yaml new file mode 100644 index 0000000000..41be014e67 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-6p1d-dep4-dep8-stp.yaml @@ -0,0 +1,158 @@ +schema: 2 +name: "disagg-b300-6p1d-dep4-dep8" + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "41882ae9b07232eed4850fb1daf8c958abb2556a" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 3 + workers: 6 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + enable-dp-lm-head: true + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + moe-a2a-backend: "megamoe" + moe-dense-tp-size: 1 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.20 + context-length: 9216 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 18432 + cuda-graph-max-bs: 1280 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "3072" + req_rate: "inf" + use_chat_template: false + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-8p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-8p1d-dep4-dep8-mtp.yaml new file mode 100644 index 0000000000..6cf0622e83 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/8k1k/disagg-8p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,148 @@ +schema: 2 +name: "disagg-b300-8p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "41882ae9b07232eed4850fb1daf8c958abb2556a" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +engine: sglang +roles: + prefill: + nodes: 4 + workers: 8 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + UCX_TLS: "cuda_copy,rc" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + watchdog-timeout: 86400 + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 2 + + mem-fraction-static: 0.85 + max-running-requests: 8192 + cuda-graph-max-bs: 1280 + swa-full-tokens-ratio: 0.1 + context-length: 9216 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "3072" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p1d-dep8-dep16-6-c512-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p1d-dep8-dep16-6-c512-stp.yaml new file mode 100644 index 0000000000..9b8c730414 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p1d-dep8-dep16-6-c512-stp.yaml @@ -0,0 +1,170 @@ +schema: 2 +name: "disagg-gb200-1p1d-dep8-dep16-6-c512" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_OPT_FP8_WO_A_GEMM: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": + {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 65536 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + load-balance-method: "total_requests" + moe-a2a-backend: "megamoe" + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.056 + context-length: 9216 + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 21504 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p1d-p-tp8-d-tp8-b8-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p1d-p-tp8-d-tp8-b8-c1-mtp.yaml new file mode 100644 index 0000000000..865e365589 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p1d-p-tp8-d-tp8-b8-c1-mtp.yaml @@ -0,0 +1,137 @@ +schema: 2 +name: "dsv4-pro-gb200-disagg-8k1k-low-latency-1p1d-tp8-tp8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 16 + cuda-graph-max-bs: 8 + chunked-prefill-size: 65536 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b1024-c1024-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b1024-c1024-mtp.yaml new file mode 100644 index 0000000000..f7965ab697 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b1024-c1024-mtp.yaml @@ -0,0 +1,156 @@ +schema: 2 +name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-1p1d-dep8-dep16-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.8 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 65536 + stream-interval: 60 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.85 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1024" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p1d-tp8-tp8-4-c1-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p1d-tp8-tp8-4-c1-stp.yaml new file mode 100644 index 0000000000..614e5ef4ca --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p1d-tp8-tp8-4-c1-stp.yaml @@ -0,0 +1,130 @@ +schema: 2 +name: "disagg-gb200-1p1d-tp8-tp8-4-c1" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.90 + max-running-requests: 1024 + cuda-graph-max-bs: 512 + chunked-prefill-size: 65536 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p2d-dep8-dep16-10-c256-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p2d-dep8-dep16-10-c256-stp.yaml new file mode 100644 index 0000000000..116927d8a3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p2d-dep8-dep16-10-c256-stp.yaml @@ -0,0 +1,170 @@ +schema: 2 +name: "disagg-gb200-1p2d-dep8-dep16-10-c256" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_OPT_FP8_WO_A_GEMM: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": + {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 65536 + + decode: + nodes: 8 + workers: 2 + gpus: 16 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + load-balance-method: "total_requests" + moe-a2a-backend: "megamoe" + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.056 + context-length: 9216 + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 21504 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "256" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p4d-dep8-tp8-10-c64-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p4d-dep8-tp8-10-c64-stp.yaml new file mode 100644 index 0000000000..2440a70869 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p4d-dep8-tp8-10-c64-stp.yaml @@ -0,0 +1,156 @@ +schema: 2 +name: "disagg-gb200-1p4d-dep8-tp8-10-c64" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_OPT_FP8_WO_A_GEMM: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": + {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 65536 + + decode: + nodes: 8 + workers: 4 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_TOPK_V2: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "64" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p6d-p-tp8-ep8-dp8-d-tp8-b128-c32x64x128-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p6d-p-tp8-ep8-dp8-d-tp8-b128-c32x64x128-mtp.yaml new file mode 100644 index 0000000000..2b55515254 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p6d-p-tp8-ep8-dp8-d-tp8-b128-c32x64x128-mtp.yaml @@ -0,0 +1,144 @@ +schema: 2 +name: "dsv4-pro-gb200-disagg-8k1k-low-latency-1p6d-dep8-tp8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.9 + max-running-requests: 256 + cuda-graph-max-bs: 128 + chunked-prefill-size: 65536 + + decode: + nodes: 12 + workers: 6 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "32x64x128" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-2p1d-dep8-dep16-8-c1536-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-2p1d-dep8-dep16-8-c1536-stp.yaml new file mode 100644 index 0000000000..986e04055c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-2p1d-dep8-dep16-8-c1536-stp.yaml @@ -0,0 +1,170 @@ +schema: 2 +name: "disagg-gb200-2p1d-dep8-dep16-8-c1536" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_OPT_FP8_WO_A_GEMM: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": + {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 65536 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + load-balance-method: "total_requests" + moe-a2a-backend: "megamoe" + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.056 + context-length: 9216 + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 21504 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1536" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-2p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b2048-c2048-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-2p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b2048-c2048-mtp.yaml new file mode 100644 index 0000000000..2529657275 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-2p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b2048-c2048-mtp.yaml @@ -0,0 +1,156 @@ +schema: 2 +name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-2p1d-dep8-dep16-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.8 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 65536 + stream-interval: 60 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.85 + max-running-requests: 2048 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "2048" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-3p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b4096-c3072-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-3p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b4096-c3072-mtp.yaml new file mode 100644 index 0000000000..ec01e22000 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-3p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b4096-c3072-mtp.yaml @@ -0,0 +1,156 @@ +schema: 2 +name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-3p1d-dep8-dep16-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 6 + workers: 3 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.8 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 65536 + stream-interval: 60 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.85 + max-running-requests: 4096 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "3072" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-4p1d-dep8-dep16-12-c4096-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-4p1d-dep8-dep16-12-c4096-stp.yaml new file mode 100644 index 0000000000..4cda0927bd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-4p1d-dep8-dep16-12-c4096-stp.yaml @@ -0,0 +1,170 @@ +schema: 2 +name: "disagg-gb200-4p1d-dep8-dep16-12-c4096" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 8 + workers: 4 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_OPT_FP8_WO_A_GEMM: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": + {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 65536 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + load-balance-method: "total_requests" + moe-a2a-backend: "megamoe" + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.056 + context-length: 9216 + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 21504 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-4p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b6144-c6144-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-4p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b6144-c6144-mtp.yaml new file mode 100644 index 0000000000..de9894359a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-4p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b6144-c6144-mtp.yaml @@ -0,0 +1,156 @@ +schema: 2 +name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-4p1d-dep8-dep16-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 8 + workers: 4 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.8 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 65536 + stream-interval: 60 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.85 + max-running-requests: 6144 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "6144" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-5p1d-dep8-dep16-14-c8192-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-5p1d-dep8-dep16-14-c8192-stp.yaml new file mode 100644 index 0000000000..91a7d7f7de --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-5p1d-dep8-dep16-14-c8192-stp.yaml @@ -0,0 +1,170 @@ +schema: 2 +name: "disagg-gb200-5p1d-dep8-dep16-14-c8192" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 10 + workers: 5 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_OPT_FP8_WO_A_GEMM: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": + {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 65536 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + load-balance-method: "total_requests" + moe-a2a-backend: "megamoe" + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.056 + context-length: 9216 + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 21504 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "8192" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-5p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b16384-c8192-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-5p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b16384-c8192-mtp.yaml new file mode 100644 index 0000000000..fbdb7547ef --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-5p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b16384-c8192-mtp.yaml @@ -0,0 +1,156 @@ +schema: 2 +name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-5p1d-dep8-dep16-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 10 + workers: 5 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.8 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 65536 + stream-interval: 60 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.85 + max-running-requests: 16384 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "8192" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-6p1d-dep8-dep12-15-c8192-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-6p1d-dep8-dep12-15-c8192-stp.yaml new file mode 100644 index 0000000000..8e63f133e0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-6p1d-dep8-dep12-15-c8192-stp.yaml @@ -0,0 +1,170 @@ +schema: 2 +name: "disagg-gb200-6p1d-dep8-dep12-15-c8192" + + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 12 + workers: 6 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + SGLANG_OPT_FP8_WO_A_GEMM: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": + {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + mem-fraction-static: 0.80 + max-running-requests: 1024 + chunked-prefill-size: 65536 + + decode: + nodes: 3 + workers: 1 + gpus: 12 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" + SGLANG_OPT_USE_FAST_MASK_EP: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + load-balance-method: "total_requests" + moe-a2a-backend: "megamoe" + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.056 + context-length: 9216 + tensor-parallel-size: 12 + data-parallel-size: 12 + expert-parallel-size: 12 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 21504 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "8192" + req_rate: "inf" + use_chat_template: false + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-6p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b21504-c16384-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-6p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b21504-c16384-mtp.yaml new file mode 100644 index 0000000000..b992c60e00 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-6p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b21504-c16384-mtp.yaml @@ -0,0 +1,156 @@ +schema: 2 +name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-6p1d-dep8-dep16-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 12 + workers: 6 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.8 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 65536 + stream-interval: 60 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.85 + max-running-requests: 21504 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "16384" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-10p1d-dep4-dep32-18-c2500-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-10p1d-dep4-dep32-18-c2500-stp.yaml new file mode 100644 index 0000000000..be505c7f86 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-10p1d-dep4-dep32-18-c2500-stp.yaml @@ -0,0 +1,169 @@ +schema: 2 +name: "disagg-gb300-10p1d-dep4-dep32-18-c2500" + +# Weiliang wide-EP sweep point: EP=32, 10P+8D = 18 nodes, conc=2500. +# Matches srt-slurm PR#173 zip_override EP=32 topology. +# Env vars and sglang_config from InferenceX main (not Weiliang's 0510 image). + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "81d0555ee23519cea80a42b4fe824e30368b7300" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 10 + workers: 10 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": + {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + enable-dp-lm-head: true + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + moe-a2a-backend: "megamoe" + + moe-dense-tp-size: 1 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.20 + context-length: 9216 + tensor-parallel-size: 32 + data-parallel-size: 32 + expert-parallel-size: 32 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 18432 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2500" + req_rate: "inf" + use_chat_template: false + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-12p1d-dep4-dep24-18-c3000-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-12p1d-dep4-dep24-18-c3000-stp.yaml new file mode 100644 index 0000000000..b19cec75d2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-12p1d-dep4-dep24-18-c3000-stp.yaml @@ -0,0 +1,169 @@ +schema: 2 +name: "disagg-gb300-12p1d-dep4-dep24-18-c3000" + +# Weiliang wide-EP sweep point: EP=24, 12P+6D = 18 nodes, conc=3000. +# Matches srt-slurm PR#173 zip_override EP=24 topology. +# Env vars and sglang_config from InferenceX main (not Weiliang's 0510 image). + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "81d0555ee23519cea80a42b4fe824e30368b7300" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 12 + workers: 12 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": + {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + enable-dp-lm-head: true + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + nodes: 6 + workers: 1 + gpus: 24 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + moe-a2a-backend: "megamoe" + + moe-dense-tp-size: 1 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.20 + context-length: 9216 + tensor-parallel-size: 24 + data-parallel-size: 24 + expert-parallel-size: 24 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 18432 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "3000" + req_rate: "inf" + use_chat_template: false + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-14p1d-dep4-dep16-18-c8192-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-14p1d-dep4-dep16-18-c8192-stp.yaml new file mode 100644 index 0000000000..6c1a1ee8e1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-14p1d-dep4-dep16-18-c8192-stp.yaml @@ -0,0 +1,169 @@ +schema: 2 +name: "disagg-gb300-14p1d-dep4-dep16-18-c8192" + +# Weiliang wide-EP sweep point: EP=16, 14P+4D = 18 nodes, conc=8192. +# Matches srt-slurm PR#173 zip_override EP=16 topology. +# Env vars and sglang_config from InferenceX main (not Weiliang's 0510 image). + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "81d0555ee23519cea80a42b4fe824e30368b7300" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 14 + workers: 14 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": + {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + enable-dp-lm-head: true + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + moe-a2a-backend: "megamoe" + + moe-dense-tp-size: 1 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.20 + context-length: 9216 + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 18432 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "8192" + req_rate: "inf" + use_chat_template: false + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-15p1d-dep4-dep12-18-c12000-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-15p1d-dep4-dep12-18-c12000-stp.yaml new file mode 100644 index 0000000000..4b248787fc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-15p1d-dep4-dep12-18-c12000-stp.yaml @@ -0,0 +1,169 @@ +schema: 2 +name: "disagg-gb300-15p1d-dep4-dep12-18-c12000" + +# Weiliang wide-EP sweep point: EP=12, 15P+3D = 18 nodes, conc=12000. +# Matches srt-slurm PR#173 zip_override EP=12 topology. +# Env vars and sglang_config from InferenceX main (not Weiliang's 0510 image). + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "81d0555ee23519cea80a42b4fe824e30368b7300" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 15 + workers: 15 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": + {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + enable-dp-lm-head: true + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + nodes: 3 + workers: 1 + gpus: 12 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + moe-a2a-backend: "megamoe" + + moe-dense-tp-size: 1 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.20 + context-length: 9216 + tensor-parallel-size: 12 + data-parallel-size: 12 + expert-parallel-size: 12 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 18432 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "12000" + req_rate: "inf" + use_chat_template: false + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-dep4-dep16-5-c1024-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-dep4-dep16-5-c1024-stp.yaml new file mode 100644 index 0000000000..cd779217b1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-dep4-dep16-5-c1024-stp.yaml @@ -0,0 +1,195 @@ +schema: 2 +name: "disagg-gb300-1p1d-dep4-dep16-5-c1024" + +# 8k/1k high-throughput topology for the wideep DSV4-Pro setup. +# +# Schema/values come from PR #1213 (513cbef) — that PR introduced the +# `dsv4-pro-gb300-fp4` upstream-style recipe with two `zip_override` +# variants (wideep [0] / narrow_ep [1]) and `backend.benchmark`. Our +# pinned srtctl (NVIDIA/srt-slurm @ sa-submission-q2-2026) doesn't +# support either: `zip_override_*_hightpt` rejects with `Unknown field` +# and `benchmark` only validates at top level. So this file inlines the +# wideep [0] override and lifts `benchmark` back out — same operational +# values, schema the pinned srtctl will accept. +# +# Other adjustments back to the InferenceX cluster shape: container & +# model.path restored to the aliases mapped in launch_gb300.sh's +# srtslurm.yaml (`lmsysorg/sglang:deepseek-v4-grace-blackwell` and +# `deepseek-v4-pro`); `dynamo.install: true` added so the container +# (which has no dynamo baked in) installs from the pinned hash. +# +# Cluster-specific items NOT inlined (require InferenceX-side equivalents): +# - slurm.partition (the source cluster uses `hpc-mid`) +# - frontend.nginx_container (yangminl's `nginx-1.27.4.sqsh` path) +# - extra_mount: yangminl/sglang-patched/sglang. Earlier diff analysis +# showed only `expert_location_dispatch.py` topk_ids int32 cast is an +# active runtime diff vs container sglang; other patched files are +# env-gated dead code under the same SGLANG_OPT_* flags this yaml +# already sets. +# +# DG-related env intentionally diverged (DG cache path is host-specific): +# - SGLANG_DG_CACHE_DIR=/configs/deepgemm_cache (yangminl host) +# - SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 (yangminl uses prebuilt cache) +# This yaml uses SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 instead. + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "81d0555ee23519cea80a42b4fe824e30368b7300" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": + {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + mem-fraction-static: 0.90 + max-running-requests: 512 + chunked-prefill-size: 32768 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + # is single-node only and corrupts results in 2-node decode setups. + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + load-balance-method: "total_requests" + moe-a2a-backend: "megamoe" + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.056 + context-length: 9216 + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 21504 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1024" + req_rate: "inf" + use_chat_template: false + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + # 9401 is already bound by the cluster-level exporter on im-gb300 nodes; + # use a port outside that range. + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-d-tp4-b8-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-d-tp4-b8-c1-mtp.yaml new file mode 100644 index 0000000000..4f67f846e5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-d-tp4-b8-c1-mtp.yaml @@ -0,0 +1,134 @@ +schema: 2 +name: "dsv4-pro-gb300-disagg-8k1k-low-latency-1p1d-tp4-tp4-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "81d0555ee23519cea80a42b4fe824e30368b7300" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" + precision: "mxfp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + chunked-prefill-size: 32768 + + decode: + nodes: 1 + workers: 1 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 + # is single-node only and corrupts results in 2-node decode setups. + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b3072-c256-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b3072-c256-mtp.yaml new file mode 100644 index 0000000000..6d4311c3a9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b3072-c256-mtp.yaml @@ -0,0 +1,154 @@ +schema: 2 +name: "dsv4-pro-gb300-disagg-8k1k-mid-curve-1p1d-dep4-dep16-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "81d0555ee23519cea80a42b4fe824e30368b7300" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" + precision: "mxfp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 256 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.94 + max-running-requests: 3072 + cuda-graph-max-bs: 256 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b3072-c256-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b3072-c256-mtp.yaml new file mode 100644 index 0000000000..b72f8c1707 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b3072-c256-mtp.yaml @@ -0,0 +1,154 @@ +schema: 2 +name: "dsv4-pro-gb300-disagg-8k1k-mid-curve-1p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "81d0555ee23519cea80a42b4fe824e30368b7300" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" + precision: "mxfp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 256 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.94 + max-running-requests: 3072 + cuda-graph-max-bs: 256 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-tp4-tp4-2-c1-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-tp4-tp4-2-c1-stp.yaml new file mode 100644 index 0000000000..a97ea9f846 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-tp4-tp4-2-c1-stp.yaml @@ -0,0 +1,177 @@ +schema: 2 +name: "disagg-gb300-1p1d-tp4-tp4-2-c1" + +# 8k/1k high-throughput topology for the wideep DSV4-Pro setup. +# +# Schema/values come from PR #1213 (513cbef) — that PR introduced the +# `dsv4-pro-gb300-fp4` upstream-style recipe with two `zip_override` +# variants (wideep [0] / narrow_ep [1]) and `backend.benchmark`. Our +# pinned srtctl (NVIDIA/srt-slurm @ sa-submission-q2-2026) doesn't +# support either: `zip_override_*_hightpt` rejects with `Unknown field` +# and `benchmark` only validates at top level. So this file inlines the +# wideep [0] override and lifts `benchmark` back out — same operational +# values, schema the pinned srtctl will accept. +# +# Other adjustments back to the InferenceX cluster shape: container & +# model.path restored to the aliases mapped in launch_gb300.sh's +# srtslurm.yaml (`lmsysorg/sglang:deepseek-v4-grace-blackwell` and +# `deepseek-v4-pro`); `dynamo.install: true` added so the container +# (which has no dynamo baked in) installs from the pinned hash. +# +# Cluster-specific items NOT inlined (require InferenceX-side equivalents): +# - slurm.partition (the source cluster uses `hpc-mid`) +# - frontend.nginx_container (yangminl's `nginx-1.27.4.sqsh` path) +# - extra_mount: yangminl/sglang-patched/sglang. Earlier diff analysis +# showed only `expert_location_dispatch.py` topk_ids int32 cast is an +# active runtime diff vs container sglang; other patched files are +# env-gated dead code under the same SGLANG_OPT_* flags this yaml +# already sets. +# +# DG-related env intentionally diverged (DG cache path is host-specific): +# - SGLANG_DG_CACHE_DIR=/configs/deepgemm_cache (yangminl host) +# - SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 (yangminl uses prebuilt cache) +# This yaml uses SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 instead. + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp4" + +# See ../1k1k/disagg-gb200-1p1d-dep8-tep8.yaml for the dynamo pin +# rationale. Hash bumped from PR #1213 to track the dynamo-sglang dsv4 +# dev branch. +dynamo: + install: true + +# Match yangminl's working all-dynamo.yaml on the source cluster: +# cpus-per-task=144 — without this slurm hands out 1 CPU/task, which +# turns the dynamo `hash:` cold source build (~500 rust crates, +# ravif/exr/zip/pyo3 stack) into a 30+ min serial compile. With 144 +# cargo finishes in ~5 min. +# mem=0 — slurm's "give the whole node's memory"; needed +# for sglang loading 671B FP4 weights + dynamo build at the same +# time without OOM. + source: + rev: "81d0555ee23519cea80a42b4fe824e30368b7300" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +# Topology: 7 prefill (TP=4 / DP=4 / EP=4 / 1 node each) + 1 decode +# (TP=8 / DP=8 / EP=8 / 2 nodes). 9 nodes total. +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + # is single-node only and corrupts results in 2-node decode setups. + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1" + req_rate: "inf" + use_chat_template: false + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + # 9401 is already bound by the cluster-level exporter on im-gb300 nodes; + # use a port outside that range. + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp4-b128-c8x32x64-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp4-b128-c8x32x64-mtp.yaml new file mode 100644 index 0000000000..3185b292c3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp4-b128-c8x32x64-mtp.yaml @@ -0,0 +1,143 @@ +schema: 2 +name: "dsv4-pro-gb300-disagg-8k1k-low-latency-1p6d-dep4-tp4-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "81d0555ee23519cea80a42b4fe824e30368b7300" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" + precision: "mxfp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + chunked-prefill-size: 32768 + + decode: + nodes: 6 + workers: 6 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 + # is single-node only and corrupts results in 2-node decode setups. + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "8x32x64" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-2p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b3072-c512-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-2p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b3072-c512-mtp.yaml new file mode 100644 index 0000000000..6656d09ce2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-2p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b3072-c512-mtp.yaml @@ -0,0 +1,154 @@ +schema: 2 +name: "dsv4-pro-gb300-disagg-8k1k-mid-curve-2p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "81d0555ee23519cea80a42b4fe824e30368b7300" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" + precision: "mxfp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 3072 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "512" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b3072-c1024-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b3072-c1024-mtp.yaml new file mode 100644 index 0000000000..2db68edeaa --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b3072-c1024-mtp.yaml @@ -0,0 +1,154 @@ +schema: 2 +name: "dsv4-pro-gb300-disagg-8k1k-mid-curve-4p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "81d0555ee23519cea80a42b4fe824e30368b7300" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" + precision: "mxfp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 4 + workers: 4 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 3072 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1024" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-8p1d-dep4-dep40-18-c2048-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-8p1d-dep4-dep40-18-c2048-stp.yaml new file mode 100644 index 0000000000..8df9a97e59 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-8p1d-dep4-dep40-18-c2048-stp.yaml @@ -0,0 +1,170 @@ +schema: 2 +name: "disagg-gb300-8p1d-dep4-dep40-18-c2048" + +# Weiliang wide-EP sweep point: EP=40, 8P+10D = 18 nodes, conc=2048. +# Matches srt-slurm PR#173 zip_override EP=40 topology. +# Env vars and sglang_config from InferenceX main (not Weiliang's 0510 image). + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp4" + +dynamo: + install: true + + source: + rev: "81d0555ee23519cea80a42b4fe824e30368b7300" +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +engine: sglang +roles: + prefill: + nodes: 8 + workers: 8 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": + {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' + moe-dense-tp-size: 1 + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + enable-dp-lm-head: true + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + nodes: 10 + workers: 1 + gpus: 40 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_LOG_FORWARD_ITERS: "1" + SGLANG_LOG_MS: "1" + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + trust-remote-code: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + + moe-a2a-backend: "megamoe" + + moe-dense-tp-size: 1 + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + disaggregation-decode-polling-interval: 8 + + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.20 + context-length: 9216 + tensor-parallel-size: 40 + data-parallel-size: 40 + expert-parallel-size: 40 + ep-num-redundant-experts: 16 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 18400 + cuda-graph-max-bs: 1280 + + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" + use_chat_template: false + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml new file mode 100644 index 0000000000..7ae0f87d04 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,154 @@ +schema: 2 +name: "dsv4-pro-gb300-disagg-8k1k-high-conc-6p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "81d0555ee23519cea80a42b4fe824e30368b7300" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" + precision: "mxfp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 6 + workers: 6 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + + mem-fraction-static: 0.9 + max-running-requests: 4096 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 9216 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "4096" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml new file mode 100644 index 0000000000..cd08b18a55 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml @@ -0,0 +1,154 @@ +schema: 2 +name: "dsv4-pro-gb300-disagg-8k1k-high-conc-8p1d-dep4-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + install: true + + source: + rev: "81d0555ee23519cea80a42b4fe824e30368b7300" +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" + precision: "mxfp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 8 + workers: 8 + gpus: 4 + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.9 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 32768 + stream-interval: 60 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_DISABLE_REUSE: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + + speculative-algo: "EAGLE" + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 2 + + mem-fraction-static: 0.85 + max-running-requests: 8192 + cuda-graph-max-bs: 1280 + swa-full-tokens-ratio: 0.1 + context-length: 9216 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "8192" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml new file mode 100644 index 0000000000..c49541b92b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml @@ -0,0 +1,130 @@ +schema: 2 +name: "agg-gb300-tp4-mtp-lowlatency" + +# Low-latency AgentX aggregate topology: one TP4 worker occupies one +# four-GPU GB300 node and serves both prefill and decode with DSpark K=6. + +model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9" + +dynamo: + install: true + source: + wheel: "1.5.0.dev20260910" + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" + enable-metrics: true + enable-cache-report: true + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.94 + page-size: 256 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + enable-deepseek-v4-fp4-indexer: true + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 32 + cuda-graph-max-bs-decode: 32 + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 4 + ep-size: 1 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml new file mode 100644 index 0000000000..0aea65fbef --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml @@ -0,0 +1,130 @@ +schema: 2 +name: "agg-gb300-tp8-mtp-lowlatency" + +# Low-latency AgentX aggregate topology: one TP8 worker spans two +# four-GPU GB300 nodes and serves both prefill and decode with DSpark K=6. + +model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9" + +dynamo: + install: true + source: + wheel: "1.5.0.dev20260910" + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +engine: sglang +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" + enable-metrics: true + enable-cache-report: true + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.94 + page-size: 256 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + enable-deepseek-v4-fp4-indexer: true + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 4 + cuda-graph-max-bs-decode: 4 + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 8 + ep-size: 1 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "8" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml new file mode 100644 index 0000000000..15885fb9af --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml @@ -0,0 +1,234 @@ +schema: 2 +name: "disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (2P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 480. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into agentic_srt.sh from the master-config conc-list. + +model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" + +dynamo: + install: true + source: + wheel: "1.5.0.dev20260902" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.85 + page-size: 256 + swa-full-tokens-ratio: 0.02 + max-running-requests: 256 + cuda-graph-max-bs: 256 + chunked-prefill-size: 65536 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 16 + dp-size: 16 + ep-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + page-size: 256 + swa-full-tokens-ratio: 0.02 + max-running-requests: 3072 + cuda-graph-max-bs: 256 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml new file mode 100644 index 0000000000..00ce318b0d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml @@ -0,0 +1,236 @@ +schema: 2 +name: "disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 +# (4P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 960. +# +# DEP8-prefill variant aligned with the measured Pareto point: prefill uses +# tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. +# Concurrency is exported into agentic_srt.sh +# from the master-config conc-list. + +model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" + +dynamo: + install: true + source: + wheel: "1.5.0.dev20260902" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +engine: sglang +roles: + prefill: + nodes: 4 + workers: 2 + env: + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + OMP_NUM_THREADS: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_DSV4_MHC_PREWARM: '1' + + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.85 + page-size: 256 + swa-full-tokens-ratio: 0.02 + max-running-requests: 256 + cuda-graph-max-bs: 256 + chunked-prefill-size: 65536 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + + decode: + nodes: 4 + workers: 1 + + + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + OMP_NUM_THREADS: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_DSV4_MHC_PREWARM: '1' + + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 16 + dp-size: 16 + ep-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + page-size: 256 + swa-full-tokens-ratio: 0.02 + max-running-requests: 3072 + cuda-graph-max-bs: 256 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml new file mode 100644 index 0000000000..f0e66a32d8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml @@ -0,0 +1,234 @@ +schema: 2 +name: "disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (6P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1440. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into agentic_srt.sh from the master-config conc-list. + +model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" + +dynamo: + install: true + source: + wheel: "1.5.0.dev20260902" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +engine: sglang +roles: + prefill: + nodes: 6 + workers: 3 + gpus: 8 + env: + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.85 + page-size: 256 + swa-full-tokens-ratio: 0.02 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 65536 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 16 + dp-size: 16 + ep-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + page-size: 256 + swa-full-tokens-ratio: 0.02 + max-running-requests: 3072 + cuda-graph-max-bs: 512 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml new file mode 100644 index 0000000000..e569bd41cd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml @@ -0,0 +1,239 @@ +schema: 2 +name: "disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 +# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1920. +# +# Uses the flat single-variant srtctl schema the agentic CI flow expects; +# resources + backend (prefill/decode env + sglang_config) are normalized +# from the Pareto run. +# Concurrency is exported into agentic_srt.sh from the master-config conc-list. + +model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" + +dynamo: + install: true + source: + wheel: "1.5.0.dev20260902" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + nginx_keepalive_timeout: "900s" + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + # AgentX warmup can legitimately keep the single wide decode worker busy + # for longer than Dynamo's 10-second TCP request-plane default. + DYN_TCP_REQUEST_TIMEOUT: "60" + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +engine: sglang +roles: + prefill: + nodes: 8 + workers: 4 + gpus: 8 + env: + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.85 + page-size: 256 + swa-full-tokens-ratio: 0.02 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 65536 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + NCCL_TIMEOUT: '100000' + NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 16 + dp-size: 16 + ep-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + page-size: 256 + swa-full-tokens-ratio: 0.02 + max-running-requests: 3072 + cuda-graph-max-bs: 192 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml new file mode 100644 index 0000000000..fb2afeea82 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml @@ -0,0 +1,141 @@ +schema: 2 +name: "agg-h200-tp8-mtp-kvoffload" + +# DeepSeek-V4-Pro AgentX aggregated TP8 serving on one 8xH200 node. The recipe is +# topology-invariant; configs/nvidia-master.yaml varies only concurrency to +# produce the latency/throughput Pareto curve. + +model: + path: "deepseek-v4-pro" + container: "dynamo-sglang" + precision: "fp8" + +dynamo: + install: true + source: + wheel: "1.3.0.dev20260718" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "h200" + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + + env: + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_SIMULATE_ACC_LEN: "2.49" + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: "1" + HF_HUB_CACHE: /hf_hub_cache + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + enable-metrics: true + trust-remote-code: true + stream-interval: 50 + watchdog-timeout: 1000000 + mem-fraction-static: 0.88 + chunked-prefill-size: 4096 + moe-runner-backend: marlin + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 32 + cuda-graph-max-bs: 32 + + scheduler-recv-interval: 30 + tp-size: 8 + dp-size: 1 + ep-size: 1 + + enable-hierarchical-cache: true + hicache-ratio: 6 + hicache-write-policy: write_through + hicache-io-backend: kernel + + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + +srun_options: + # The custom benchmark installs its isolated AIPerf environment at runtime. + container-remap-root: "" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + +benchmark: + type: custom + # Replaced with CONC_LIST by the launcher before submission. + concurrencies: [1] + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + # Aggregated serving uses one TP8 worker for both prefill and decode. + IS_MULTINODE: "true" + TP: "8" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + # Fail the run loudly if the aggregate engine's sglang: metrics never reach + # the AIPerf export, instead of publishing a trace artifact with no backend + # series behind it. + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep4-dep16-b64-eplb384-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep4-dep16-b64-eplb384-mtp3.yaml new file mode 100644 index 0000000000..2b15a2f46f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep4-dep16-b64-eplb384-mtp3.yaml @@ -0,0 +1,154 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx10dep4_gen1dep16_batch64_eplb384_mtp3" + +# ctx: 10 prefill workers, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP16/EP16, enable_attention_dp=true, max_batch=64 +# concurrency: 1229 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 10 + workers: 10 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + tokens_per_block: 128 + max_batch_size: 64 + max_num_tokens: 256 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep16_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 16 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1229" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: true + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep4-dep8-b512-eplb384-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep4-dep8-b512-eplb384-mtp0.yaml new file mode 100644 index 0000000000..d663486d9b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep4-dep8-b512-eplb384-mtp0.yaml @@ -0,0 +1,204 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx10dep4_gen1dep8_batch512_eplb384_mtp0" + +# ctx: 10 prefill workers, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP8/EP8, enable_attention_dp=true, max_batch=512 +# concurrency: 4301 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 10 + workers: 10 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + tensor_parallel_size: 4 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + - 264 + - 272 + - 280 + - 288 + - 296 + - 304 + - 312 + - 320 + - 328 + - 336 + - 344 + - 352 + - 360 + - 368 + - 376 + - 384 + - 392 + - 400 + - 408 + - 416 + - 424 + - 432 + - 440 + - 448 + - 456 + - 464 + - 472 + - 480 + - 488 + - 496 + - 504 + - 512 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 512 + max_num_tokens: 512 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep8_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + stream_interval: 100 + tensor_parallel_size: 8 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4301" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: true + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-12p1d-dep4-dep8-b512-eplb384-mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-12p1d-dep4-dep8-b512-eplb384-mtp1.yaml new file mode 100644 index 0000000000..e11705672b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-12p1d-dep4-dep8-b512-eplb384-mtp1.yaml @@ -0,0 +1,210 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx12dep4_gen1dep8_batch512_eplb384_mtp1" + +# ctx: 12 prefill workers, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP8/EP8, enable_attention_dp=true, max_batch=512 +# concurrency: 4301 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 12 + workers: 12 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 1 + tensor_parallel_size: 4 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + - 264 + - 272 + - 280 + - 288 + - 296 + - 304 + - 312 + - 320 + - 328 + - 336 + - 344 + - 352 + - 360 + - 368 + - 376 + - 384 + - 392 + - 400 + - 408 + - 416 + - 424 + - 432 + - 440 + - 448 + - 456 + - 464 + - 472 + - 480 + - 488 + - 496 + - 504 + - 512 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 512 + max_num_tokens: 1024 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep8_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 1 + stream_interval: 100 + tensor_parallel_size: 8 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4301" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: true + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p1d-dep4-dep32-b4-eplb384-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p1d-dep4-dep32-b4-eplb384-mtp0.yaml new file mode 100644 index 0000000000..be5fa3a274 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p1d-dep4-dep32-b4-eplb384-mtp0.yaml @@ -0,0 +1,140 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx1dep4_gen1dep32_batch4_eplb384_mtp0" + +# ctx: 1 prefill worker, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP32/EP32, enable_attention_dp=true, max_batch=4 +# concurrency: 154 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + tensor_parallel_size: 4 + + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + tokens_per_block: 128 + max_batch_size: 4 + max_num_tokens: 4 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep32_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + stream_interval: 100 + tensor_parallel_size: 32 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "154" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p4d-dep4-tep8-b1-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p4d-dep4-tep8-b1-eplb0-mtp0.yaml new file mode 100644 index 0000000000..6da9ef4fd1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p4d-dep4-tep8-b1-eplb0-mtp0.yaml @@ -0,0 +1,133 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx1dep4_gen4tep8_batch1_eplb0_mtp0" + +# ctx: 1 prefill worker, TP4/EP4 +# gen: 4 decode workers, TP8/EP8, max_batch=1 +# concurrency: 4 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + tensor_parallel_size: 4 + + decode: + nodes: 8 + workers: 4 + gpus: 8 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + stream_interval: 100 + tensor_parallel_size: 8 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p4d-dep4-tep8-b1-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p4d-dep4-tep8-b1-eplb0-mtp3.yaml new file mode 100644 index 0000000000..324ede69db --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p4d-dep4-tep8-b1-eplb0-mtp3.yaml @@ -0,0 +1,139 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx1dep4_gen4tep8_batch1_eplb0_mtp3" + +# ctx: 1 prefill worker, TP4/EP4 +# gen: 4 decode workers, TP8/EP8, max_batch=1 +# concurrency: 8 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + + decode: + nodes: 8 + workers: 4 + gpus: 8 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "8" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp0.yaml new file mode 100644 index 0000000000..699ef1955d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp0.yaml @@ -0,0 +1,133 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch1_eplb0_mtp0" + +# ctx: 1 prefill worker, TP4/EP4 +# gen: 5 decode workers, TP4/EP4, max_batch=1 +# concurrency: 5 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + tensor_parallel_size: 4 + + decode: + nodes: 5 + workers: 5 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + stream_interval: 100 + tensor_parallel_size: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "5" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp3.yaml new file mode 100644 index 0000000000..324165fb9c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp3.yaml @@ -0,0 +1,139 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch1_eplb0_mtp3" + +# ctx: 1 prefill worker, TP4/EP4 +# gen: 5 decode workers, TP4/EP4, max_batch=1 +# concurrency: 10 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + + decode: + nodes: 5 + workers: 5 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "10" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b2-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b2-eplb0-mtp0.yaml new file mode 100644 index 0000000000..45442281b8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b2-eplb0-mtp0.yaml @@ -0,0 +1,133 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch2_eplb0_mtp0" + +# ctx: 1 prefill worker, TP4/EP4 +# gen: 5 decode workers, TP4/EP4, max_batch=2 +# concurrency: 15 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + tensor_parallel_size: 4 + + decode: + nodes: 5 + workers: 5 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 2 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + stream_interval: 100 + tensor_parallel_size: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "15" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b2-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b2-eplb0-mtp3.yaml new file mode 100644 index 0000000000..a454ab46b7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b2-eplb0-mtp3.yaml @@ -0,0 +1,137 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch2_eplb0_mtp3" + +# ctx: 1 prefill worker, TP4/EP4 +# gen: 5 decode workers, TP4/EP4, max_batch=2 +# concurrency: 15 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + + decode: + nodes: 5 + workers: 5 + gpus: 4 + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "15" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp0.yaml new file mode 100644 index 0000000000..ee682beaea --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp0.yaml @@ -0,0 +1,133 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch4_eplb0_mtp0" + +# ctx: 1 prefill worker, TP4/EP4 +# gen: 5 decode workers, TP4/EP4, max_batch=4 +# concurrency: 25 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + tensor_parallel_size: 4 + + decode: + nodes: 5 + workers: 5 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 4 + max_num_tokens: 4 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + stream_interval: 100 + tensor_parallel_size: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "25" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp3.yaml new file mode 100644 index 0000000000..bce7e2884c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp3.yaml @@ -0,0 +1,139 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch4_eplb0_mtp3" + +# ctx: 1 prefill worker, TP4/EP4 +# gen: 5 decode workers, TP4/EP4, max_batch=4 +# concurrency: 30 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + + decode: + nodes: 5 + workers: 5 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "30" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b8-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b8-eplb0-mtp0.yaml new file mode 100644 index 0000000000..2cd11dbf61 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b8-eplb0-mtp0.yaml @@ -0,0 +1,134 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch8_eplb0_mtp0" + +# ctx: 1 prefill worker, TP4/EP4 +# gen: 5 decode workers, TP4/EP4, max_batch=8 +# concurrency: 55 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + tensor_parallel_size: 4 + + decode: + nodes: 5 + workers: 5 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + tokens_per_block: 128 + max_batch_size: 8 + max_num_tokens: 8 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + stream_interval: 100 + tensor_parallel_size: 4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "55" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-2p1d-dep4-dep32-b2-eplb384-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-2p1d-dep4-dep32-b2-eplb384-mtp3.yaml new file mode 100644 index 0000000000..0a4a77f69d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-2p1d-dep4-dep32-b2-eplb384-mtp3.yaml @@ -0,0 +1,146 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx2dep4_gen1dep32_batch2_eplb384_mtp3" + +# ctx: 2 prefill workers, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP32/EP32, enable_attention_dp=true, max_batch=2 +# concurrency: 84 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep32_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 32 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "84" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-2p1d-dep4-dep32-b8-eplb384-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-2p1d-dep4-dep32-b8-eplb384-mtp0.yaml new file mode 100644 index 0000000000..5c7ef5c965 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-2p1d-dep4-dep32-b8-eplb384-mtp0.yaml @@ -0,0 +1,141 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx2dep4_gen1dep32_batch8_eplb384_mtp0" + +# ctx: 2 prefill workers, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP32/EP32, enable_attention_dp=true, max_batch=8 +# concurrency: 308 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + tensor_parallel_size: 4 + + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + tokens_per_block: 128 + max_batch_size: 8 + max_num_tokens: 8 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep32_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + stream_interval: 100 + tensor_parallel_size: 32 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "308" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: true + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-3p1d-dep4-dep32-b4-eplb384-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-3p1d-dep4-dep32-b4-eplb384-mtp3.yaml new file mode 100644 index 0000000000..9ad8cbbfea --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-3p1d-dep4-dep32-b4-eplb384-mtp3.yaml @@ -0,0 +1,146 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx3dep4_gen1dep32_batch4_eplb384_mtp3" + +# ctx: 3 prefill workers, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP32/EP32, enable_attention_dp=true, max_batch=4 +# concurrency: 180 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 3 + workers: 3 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + tokens_per_block: 128 + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep32_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 32 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "180" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep32-b16-eplb384-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep32-b16-eplb384-mtp0.yaml new file mode 100644 index 0000000000..8330aa3ad4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep32-b16-eplb384-mtp0.yaml @@ -0,0 +1,142 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx4dep4_gen1dep32_batch16_eplb384_mtp0" + +# ctx: 4 prefill workers, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP32/EP32, enable_attention_dp=true, max_batch=16 +# concurrency: 615 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 4 + workers: 4 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + tensor_parallel_size: 4 + + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + tokens_per_block: 128 + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep32_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + stream_interval: 100 + tensor_parallel_size: 32 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "615" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: true + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep32-b8-eplb384-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep32-b8-eplb384-mtp3.yaml new file mode 100644 index 0000000000..ab337798b3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep32-b8-eplb384-mtp3.yaml @@ -0,0 +1,147 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx4dep4_gen1dep32_batch8_eplb384_mtp3" + +# ctx: 4 prefill workers, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP32/EP32, enable_attention_dp=true, max_batch=8 +# concurrency: 333 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 4 + workers: 4 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + tokens_per_block: 128 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep32_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 32 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "333" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: true + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-5p1d-dep4-dep16-b64-eplb384-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-5p1d-dep4-dep16-b64-eplb384-mtp0.yaml new file mode 100644 index 0000000000..ae39b54307 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-5p1d-dep4-dep16-b64-eplb384-mtp0.yaml @@ -0,0 +1,148 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx5dep4_gen1dep16_batch64_eplb384_mtp0" + +# ctx: 5 prefill workers, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP16/EP16, enable_attention_dp=true, max_batch=64 +# concurrency: 1229 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 5 + workers: 5 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + tensor_parallel_size: 4 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + tokens_per_block: 128 + max_batch_size: 64 + max_num_tokens: 64 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep16_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + stream_interval: 100 + tensor_parallel_size: 16 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1229" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: true + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-6p1d-dep4-dep16-b32-eplb384-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-6p1d-dep4-dep16-b32-eplb384-mtp3.yaml new file mode 100644 index 0000000000..d259ea172f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-6p1d-dep4-dep16-b32-eplb384-mtp3.yaml @@ -0,0 +1,150 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx6dep4_gen1dep16_batch32_eplb384_mtp3" + +# ctx: 6 prefill workers, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP16/EP16, enable_attention_dp=true, max_batch=32 +# concurrency: 666 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 6 + workers: 6 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + tokens_per_block: 128 + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep16_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 16 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "666" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: true + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-6p1d-dep4-dep32-b32-eplb384-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-6p1d-dep4-dep32-b32-eplb384-mtp0.yaml new file mode 100644 index 0000000000..63598d4207 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-6p1d-dep4-dep32-b32-eplb384-mtp0.yaml @@ -0,0 +1,144 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx6dep4_gen1dep32_batch32_eplb384_mtp0" + +# ctx: 6 prefill workers, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP32/EP32, enable_attention_dp=true, max_batch=32 +# concurrency: 1127 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 6 + workers: 6 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + tensor_parallel_size: 4 + + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + tokens_per_block: 128 + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep32_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + stream_interval: 100 + tensor_parallel_size: 32 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1127" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: true + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-6p1d-dep4-dep8-b256-eplb384-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-6p1d-dep4-dep8-b256-eplb384-mtp0.yaml new file mode 100644 index 0000000000..622b89b456 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-6p1d-dep4-dep8-b256-eplb384-mtp0.yaml @@ -0,0 +1,172 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx6dep4_gen1dep8_batch256_eplb384_mtp0" + +# ctx: 6 prefill workers, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP8/EP8, enable_attention_dp=true, max_batch=256 +# concurrency: 2253 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 6 + workers: 6 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + tensor_parallel_size: 4 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 256 + max_num_tokens: 256 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep8_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + stream_interval: 100 + tensor_parallel_size: 8 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2253" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: true + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep4-dep8-b128-eplb384-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep4-dep8-b128-eplb384-mtp3.yaml new file mode 100644 index 0000000000..1854502eb9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep4-dep8-b128-eplb384-mtp3.yaml @@ -0,0 +1,162 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx7dep4_gen1dep8_batch128_eplb384_mtp3" + +# ctx: 7 prefill workers, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP8/EP8, enable_attention_dp=true, max_batch=128 +# concurrency: 1229 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 7 + workers: 7 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 128 + max_num_tokens: 512 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep8_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1229" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: true + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-8p1d-dep4-dep32-b16-eplb384-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-8p1d-dep4-dep32-b16-eplb384-mtp3.yaml new file mode 100644 index 0000000000..3019899bb9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-8p1d-dep4-dep32-b16-eplb384-mtp3.yaml @@ -0,0 +1,148 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx8dep4_gen1dep32_batch16_eplb384_mtp3" + +# ctx: 8 prefill workers, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP32/EP32, enable_attention_dp=true, max_batch=16 +# concurrency: 615 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 8 + workers: 8 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + tokens_per_block: 128 + max_batch_size: 16 + max_num_tokens: 64 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep32_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 32 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "615" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: true + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep4-dep16-b128-eplb384-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep4-dep16-b128-eplb384-mtp0.yaml new file mode 100644 index 0000000000..eaf702173b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep4-dep16-b128-eplb384-mtp0.yaml @@ -0,0 +1,156 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx9dep4_gen1dep16_batch128_eplb384_mtp0" + +# ctx: 9 prefill workers, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP16/EP16, enable_attention_dp=true, max_batch=128 +# concurrency: 2253 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 9 + workers: 9 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + tensor_parallel_size: 4 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + tokens_per_block: 128 + max_batch_size: 128 + max_num_tokens: 128 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep16_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + stream_interval: 100 + tensor_parallel_size: 16 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2253" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: true + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep4-dep8-b256-eplb384-mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep4-dep8-b256-eplb384-mtp1.yaml new file mode 100644 index 0000000000..9d85eccc63 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep4-dep8-b256-eplb384-mtp1.yaml @@ -0,0 +1,178 @@ +schema: 2 +name: "dsv4pro_mxfp4_ISL8K_OSL1K_ctx9dep4_gen1dep8_batch256_eplb384_mtp1" + +# ctx: 9 prefill workers, TP4/EP4, EPLB: num_slots=384 +# gen: 1 decode worker, TP8/EP8, enable_attention_dp=true, max_batch=256 +# concurrency: 2253 + +model: + path: "deepseek-ai/DeepSeek-V4-Pro" + container: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1" + precision: "fp4" + +extra_mount: + - "tmpfs:/dev/shm:size=100%" + - "configs/dsv4-moe-load-balancer-configs:/dsv4-eplb-configs" +resources: + gpu_type: "gb300" + + gpus_per_node: 4 + +engine: trtllm +roles: + prefill: + nodes: 9 + workers: 9 + gpus: 4 + + env: + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: "1" + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "-1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + tokens_per_block: 128 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 8232 + moe_config: + backend: TRTLLM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_ctx_ep4_384.yaml" + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 1 + tensor_parallel_size: 4 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,sm,self,tcp" + PYTHONUNBUFFERED: "1" + ENROOT_ALLOW_DEV: "yes" + MIMALLOC_PURGE_DELAY: "0" + NCCL_GRAPH_MIXING_SUPPORT: "0" + PYTHONWARNINGS: "ignore::DeprecationWarning:cutlass.cute.core" + TLLM_LOG_LEVEL: "INFO" + TRTLLM_ENABLE_PDL: "1" + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 8192 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + tokens_per_block: 128 + max_batch_size: 256 + max_num_tokens: 512 + max_seq_len: 9256 + moe_config: + backend: MEGAMOE_DEEPGEMM + load_balancer: "/dsv4-eplb-configs/moe_load_balancer_gen_ep8_slots384.yaml" + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + trust_remote_code: true + speculative_config: + decoding_type: MTP + max_draft_len: 1 + stream_interval: 100 + tensor_parallel_size: 8 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2253" + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 20 + custom_tokenizer: "deepseek_v4" + use_chat_template: true + +frontend: + type: "dynamo" + enable_multiple_frontends: true + +health_check: + max_attempts: 720 + interval_seconds: 10 + +dynamo: + request_plane: "tcp" + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml new file mode 100644 index 0000000000..4b84c78cc5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml @@ -0,0 +1,197 @@ +schema: 2 +name: dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp +model: + path: deepseek-ai/DeepSeek-V4-Pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 +identity: + model: + repo: deepseek-ai/DeepSeek-V4-Pro + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + request_plane: tcp +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + OMP_NUM_THREADS: '1' + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_DEFAULT_THINKING_MODE: disabled + MIMALLOC_ARENA_RESERVE: '0' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + CUDA_SCALE_LAUNCH_QUEUES: 4x + args: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + custom_tokenizer: deepseek_v4 + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 193273528320 + pool_ratio: + - 0.55 + - 0.22 + - 0.23 + tokens_per_block: 128 + block_reuse_config: + policy: per_conversation + max_num_turns: 5 + max_batch_size: 256 + max_num_tokens: 16384 + max_seq_len: 990016 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 8 + decode: + nodes: 8 + workers: 1 + gpus: 32 + env: + OMP_NUM_THREADS: '1' + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_DEFAULT_THINKING_MODE: disabled + MIMALLOC_ARENA_RESERVE: '0' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + custom_tokenizer: deepseek_v4 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 200000 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.7 + host_cache_size: 0 + tokens_per_block: 128 + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 996595 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + MODEL_PREFIX: dsv4 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '388' + DURATION: '3600' + KV_OFFLOADING: dram + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + placement: + node: first_decode +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + MODEL_PREFIX: dsv4 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '388' + DURATION: '3600' + KV_OFFLOADING: dram + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: DeepSeek-V4-Pro + placement: + node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml new file mode 100644 index 0000000000..b53ac5ec78 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml @@ -0,0 +1,197 @@ +schema: 2 +name: dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp +model: + path: deepseek-ai/DeepSeek-V4-Pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 +identity: + model: + repo: deepseek-ai/DeepSeek-V4-Pro + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + request_plane: tcp +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + OMP_NUM_THREADS: '1' + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_DEFAULT_THINKING_MODE: disabled + MIMALLOC_ARENA_RESERVE: '0' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + CUDA_SCALE_LAUNCH_QUEUES: 4x + args: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + custom_tokenizer: deepseek_v4 + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 193273528320 + pool_ratio: + - 0.55 + - 0.22 + - 0.23 + tokens_per_block: 128 + block_reuse_config: + policy: per_conversation + max_num_turns: 5 + max_batch_size: 128 + max_num_tokens: 4096 + max_seq_len: 990016 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + nodes: 8 + workers: 4 + gpus: 8 + env: + OMP_NUM_THREADS: '1' + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_DEFAULT_THINKING_MODE: disabled + MIMALLOC_ARENA_RESERVE: '0' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + custom_tokenizer: deepseek_v4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 200000 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.9 + host_cache_size: 0 + tokens_per_block: 128 + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 996595 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 8 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + MODEL_PREFIX: dsv4 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '4' + DURATION: '3600' + KV_OFFLOADING: dram + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + placement: + node: first_decode +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + MODEL_PREFIX: dsv4 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '4' + DURATION: '3600' + KV_OFFLOADING: dram + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: DeepSeek-V4-Pro + placement: + node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml new file mode 100644 index 0000000000..e7734cdf6f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml @@ -0,0 +1,197 @@ +schema: 2 +name: dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp +model: + path: deepseek-ai/DeepSeek-V4-Pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 +identity: + model: + repo: deepseek-ai/DeepSeek-V4-Pro + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + request_plane: tcp +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + OMP_NUM_THREADS: '1' + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_DEFAULT_THINKING_MODE: disabled + MIMALLOC_ARENA_RESERVE: '0' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + CUDA_SCALE_LAUNCH_QUEUES: 4x + args: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + custom_tokenizer: deepseek_v4 + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 193273528320 + pool_ratio: + - 0.55 + - 0.22 + - 0.23 + tokens_per_block: 128 + block_reuse_config: + policy: per_conversation + max_num_turns: 5 + max_batch_size: 128 + max_num_tokens: 4096 + max_seq_len: 990016 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + nodes: 6 + workers: 6 + gpus: 4 + env: + OMP_NUM_THREADS: '1' + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_DEFAULT_THINKING_MODE: disabled + MIMALLOC_ARENA_RESERVE: '0' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + custom_tokenizer: deepseek_v4 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 200000 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.9 + host_cache_size: 0 + tokens_per_block: 128 + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 996595 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + MODEL_PREFIX: dsv4 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '24' + DURATION: '3600' + KV_OFFLOADING: dram + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + placement: + node: first_decode +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + MODEL_PREFIX: dsv4 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '24' + DURATION: '3600' + KV_OFFLOADING: dram + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: DeepSeek-V4-Pro + placement: + node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml new file mode 100644 index 0000000000..5cd44b00c9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml @@ -0,0 +1,198 @@ +schema: 2 +name: dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp +model: + path: deepseek-ai/DeepSeek-V4-Pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 +identity: + model: + repo: deepseek-ai/DeepSeek-V4-Pro + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + request_plane: tcp +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false +roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + env: + OMP_NUM_THREADS: '1' + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_DEFAULT_THINKING_MODE: disabled + MIMALLOC_ARENA_RESERVE: '0' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + CUDA_SCALE_LAUNCH_QUEUES: 4x + args: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + custom_tokenizer: deepseek_v4 + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 193273528320 + pool_ratio: + - 0.55 + - 0.22 + - 0.23 + tokens_per_block: 128 + block_reuse_config: + policy: per_conversation + max_num_turns: 5 + max_batch_size: 256 + max_num_tokens: 16384 + max_seq_len: 990016 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 8 + decode: + nodes: 8 + workers: 1 + gpus: 32 + env: + OMP_NUM_THREADS: '1' + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_DEFAULT_THINKING_MODE: disabled + MIMALLOC_ARENA_RESERVE: '0' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + custom_tokenizer: deepseek_v4 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 200000 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.7 + host_cache_size: 0 + tokens_per_block: 128 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 996595 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 32 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + MODEL_PREFIX: dsv4 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '736' + DURATION: '3600' + KV_OFFLOADING: dram + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + placement: + node: first_decode +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + MODEL_PREFIX: dsv4 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '736' + DURATION: '3600' + KV_OFFLOADING: dram + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: DeepSeek-V4-Pro + placement: + node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml new file mode 100644 index 0000000000..c6783fcf94 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml @@ -0,0 +1,204 @@ +schema: 2 +name: dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp +model: + path: deepseek-ai/DeepSeek-V4-Pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 +identity: + model: + repo: deepseek-ai/DeepSeek-V4-Pro + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + request_plane: tcp +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false +roles: + prefill: + nodes: 6 + workers: 3 + gpus: 8 + env: + OMP_NUM_THREADS: '1' + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_DEFAULT_THINKING_MODE: disabled + MIMALLOC_ARENA_RESERVE: '0' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + CUDA_SCALE_LAUNCH_QUEUES: 4x + args: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + custom_tokenizer: deepseek_v4 + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 193273528320 + pool_ratio: + - 0.55 + - 0.22 + - 0.23 + tokens_per_block: 128 + block_reuse_config: + policy: per_conversation + max_num_turns: 5 + max_batch_size: 256 + max_num_tokens: 16384 + max_seq_len: 990016 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 8 + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + OMP_NUM_THREADS: '1' + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_DEFAULT_THINKING_MODE: disabled + MIMALLOC_ARENA_RESERVE: '0' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + - 20 + - 24 + - 28 + - 32 + enable_padding: true + custom_tokenizer: deepseek_v4 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 200000 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.7 + host_cache_size: 0 + tokens_per_block: 128 + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 996595 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 16 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + MODEL_PREFIX: dsv4 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '1152' + DURATION: '3600' + KV_OFFLOADING: dram + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + placement: + node: first_decode +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + MODEL_PREFIX: dsv4 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '1152' + DURATION: '3600' + KV_OFFLOADING: dram + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: DeepSeek-V4-Pro + placement: + node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml new file mode 100644 index 0000000000..c89f7a97c4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml @@ -0,0 +1,220 @@ +schema: 2 +name: dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp +model: + path: deepseek-ai/DeepSeek-V4-Pro + container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 +identity: + model: + repo: deepseek-ai/DeepSeek-V4-Pro + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + request_plane: tcp +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false +roles: + prefill: + nodes: 10 + workers: 5 + gpus: 8 + env: + OMP_NUM_THREADS: '1' + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_DEFAULT_THINKING_MODE: disabled + MIMALLOC_ARENA_RESERVE: '0' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + CUDA_SCALE_LAUNCH_QUEUES: 4x + args: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + custom_tokenizer: deepseek_v4 + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 193273528320 + pool_ratio: + - 0.55 + - 0.22 + - 0.23 + tokens_per_block: 128 + block_reuse_config: + policy: per_conversation + max_num_turns: 5 + max_batch_size: 256 + max_num_tokens: 16384 + max_seq_len: 990016 + moe_config: + backend: TRTLLM + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 8 + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + OMP_NUM_THREADS: '1' + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_DEFAULT_THINKING_MODE: disabled + MIMALLOC_ARENA_RESERVE: '0' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + - 20 + - 24 + - 28 + - 32 + - 36 + - 40 + - 44 + - 48 + - 52 + - 56 + - 60 + - 64 + - 68 + - 72 + - 76 + - 80 + - 84 + - 88 + - 92 + - 96 + enable_padding: true + custom_tokenizer: deepseek_v4 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 200000 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.7 + host_cache_size: 0 + tokens_per_block: 128 + max_batch_size: 96 + max_num_tokens: 384 + max_seq_len: 996595 + moe_config: + backend: MEGAMOE_DEEPGEMM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + sparse_attention_config: + algorithm: deepseek_v4 + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 16 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + MODEL_PREFIX: dsv4 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '2626' + DURATION: '3600' + KV_OFFLOADING: dram + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + placement: + node: first_decode +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + MODEL: deepseek-ai/DeepSeek-V4-Pro + MODEL_PREFIX: dsv4 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '2626' + DURATION: '3600' + KV_OFFLOADING: dram + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: DeepSeek-V4-Pro + placement: + node: last_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..72ff21fda9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-dep8-dep8-mtp.yaml @@ -0,0 +1,152 @@ +schema: 2 +name: "svf-vllm-disagg-b200-1p1d-dep8-dep8-mtp" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +health_check: + max_attempts: 90 + interval_seconds: 10 +sbatch_directives: + segment: "1" +resources: + gpu_type: "b200" + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids + connector: +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NVSHMEM_INFO: "1" + NVSHMEM_DEBUG: "INFO" + NCCL_NVLS_ENABLE: "1" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + numa-bind: false + tokenizer-mode: deepseek_v4 + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NCCL_NVLS_ENABLE: "1" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-dep8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-dep8-tp8-mtp.yaml new file mode 100644 index 0000000000..787fb2f7f3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-dep8-tp8-mtp.yaml @@ -0,0 +1,137 @@ +schema: 2 +name: "svf-vllm-disagg-b200-1p1d-dep8-tp8-mtp" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +sbatch_directives: + segment: "1" +health_check: + max_attempts: 90 + interval_seconds: 10 +resources: + gpu_type: "b200" + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids + connector: +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NVSHMEM_INFO: "1" + NVSHMEM_DEBUG: "INFO" + NCCL_NVLS_ENABLE: "1" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enforce-eager: true + max-model-len: 9280 + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + # numa-bind: true + tokenizer-mode: deepseek_v4 + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + max-model-len: 16384 + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + max-num-seqs: 256 + max-cudagraph-capture-size: 64 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x8x16" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c256-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c256-stp.yaml new file mode 100644 index 0000000000..bd7d5b4de3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c256-stp.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: "svf-vllm-disagg-b200-low-middle-c256" +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.23.0" + precision: "fp4" +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" +setup_script: vllm-container-deps.sh +slurm: + time_limit: "8:00:00" +health_check: + interval_seconds: 10 + max_attempts: 1440 +sbatch_directives: + segment: "1" +resources: + gpu_type: b200 + gpus_per_node: 8 +frontend: + enable_multiple_frontends: false + type: dynamo +engine: + type: vllm + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids + connector: +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + NCCL_CUMEM_ENABLE: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TILELANG_CLEANUP_TEMP_FILES: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' + VLLM_SERVER_DEV_MODE: '1' + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' + args: + block-size: 256 + data-parallel-rpc-port: 13345 + data-parallel-size: 8 + enable-ep-weight-filter: true + enable-expert-parallel: true + enable-sleep-mode: true + enforce-eager: true + gpu-memory-utilization: 0.95 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + max-model-len: 9280 + max-num-batched-tokens: 32768 + max-num-seqs: 16 + no-async-scheduling: true + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + no-enable-prefix-caching: true + numa-bind: true + pipeline-parallel-size: 1 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + tensor-parallel-size: 1 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + NCCL_CUMEM_ENABLE: '1' + TILELANG_CLEANUP_TEMP_FILES: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' + VLLM_SERVER_DEV_MODE: '1' + args: + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + data-parallel-rpc-port: 13345 + data-parallel-size: 8 + enable-ep-weight-filter: true + enable-expert-parallel: true + enable-sleep-mode: true + gpu-memory-utilization: 0.9 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + max-cudagraph-capture-size: 256 + max-model-len: 9280 + max-num-batched-tokens: 256 + max-num-seqs: 256 + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + no-enable-prefix-caching: true + pipeline-parallel-size: 1 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + stream-interval: 50 + tensor-parallel-size: 1 + tokenizer-mode: deepseek_v4 + trust-remote-code: true +benchmark: + concurrencies: "256" + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + isl: 8192 + osl: 1024 + req_rate: "inf" + type: "sa-bench" + use_chat_template: true +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.23.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.23.0" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c512-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c512-stp.yaml new file mode 100644 index 0000000000..76978e4a8e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c512-stp.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: "svf-vllm-disagg-b200-low-middle-c512" +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.23.0" + precision: "fp4" +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" +setup_script: vllm-container-deps.sh +slurm: + time_limit: "8:00:00" +health_check: + max_attempts: 1440 + interval_seconds: 10 +sbatch_directives: + segment: "1" +resources: + gpu_type: b200 + gpus_per_node: 8 +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids + connector: +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + TILELANG_CLEANUP_TEMP_FILES: '1' + NCCL_CUMEM_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 32768 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + numa-bind: true + tokenizer-mode: deepseek_v4 + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + TILELANG_CLEANUP_TEMP_FILES: '1' + NCCL_CUMEM_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.23.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.23.0" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c1-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c1-stp.yaml new file mode 100644 index 0000000000..5b8c2a447f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c1-stp.yaml @@ -0,0 +1,137 @@ +schema: 2 +name: "svf-vllm-disagg-b200-low-latency-c1" +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.23.0" + precision: "fp4" +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" +setup_script: vllm-container-deps.sh +slurm: + time_limit: "8:00:00" +health_check: + interval_seconds: 10 + max_attempts: 1440 +sbatch_directives: + segment: "1" +resources: + gpu_type: b200 + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + +frontend: + enable_multiple_frontends: false + type: dynamo +engine: + type: vllm + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids + connector: +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + NCCL_CUMEM_ENABLE: '1' + TILELANG_CLEANUP_TEMP_FILES: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' + VLLM_SERVER_DEV_MODE: '1' + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' + args: + block-size: 256 + data-parallel-rpc-port: 13345 + data-parallel-size: 8 + enable-expert-parallel: true + enable-sleep-mode: true + enforce-eager: true + gpu-memory-utilization: 0.8 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + max-model-len: 9280 + max-num-batched-tokens: 32768 + max-num-seqs: 16 + no-async-scheduling: true + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + no-enable-prefix-caching: true + numa-bind: true + offload-group-size: 3 + offload-num-in-group: 1 + offload-prefetch-step: 2 + pipeline-parallel-size: 1 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + tensor-parallel-size: 1 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + NCCL_CUMEM_ENABLE: '1' + TILELANG_CLEANUP_TEMP_FILES: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_SERVER_DEV_MODE: '1' + args: + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + enable-sleep-mode: true + gpu-memory-utilization: 0.9 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + max-cudagraph-capture-size: 256 + max-model-len: 9280 + max-num-batched-tokens: 256 + max-num-seqs: 256 + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + no-enable-prefix-caching: true + pipeline-parallel-size: 1 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + stream-interval: 50 + tensor-parallel-size: 8 + tokenizer-mode: deepseek_v4 + trust-remote-code: true +benchmark: + concurrencies: "1" + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + isl: 8192 + osl: 1024 + req_rate: "inf" + type: "sa-bench" + use_chat_template: true +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.23.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.23.0" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c32x128-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c32x128-stp.yaml new file mode 100644 index 0000000000..c3d2299ee8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c32x128-stp.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: "svf-vllm-disagg-b200-low-latency-c32-c128" +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.23.0" + precision: "fp4" +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" +setup_script: vllm-container-deps.sh +slurm: + time_limit: "8:00:00" +health_check: + interval_seconds: 10 + max_attempts: 1440 +sbatch_directives: + segment: "1" +resources: + gpu_type: b200 + gpus_per_node: 8 +frontend: + enable_multiple_frontends: false + type: dynamo +engine: + type: vllm + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids + connector: +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + NCCL_CUMEM_ENABLE: '1' + TILELANG_CLEANUP_TEMP_FILES: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' + VLLM_SERVER_DEV_MODE: '1' + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' + args: + block-size: 256 + data-parallel-rpc-port: 13345 + data-parallel-size: 8 + enable-expert-parallel: true + enable-sleep-mode: true + enforce-eager: true + gpu-memory-utilization: 0.8 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + max-model-len: 9280 + max-num-batched-tokens: 32768 + max-num-seqs: 16 + no-async-scheduling: true + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + no-enable-prefix-caching: true + numa-bind: true + offload-group-size: 3 + offload-num-in-group: 1 + offload-prefetch-step: 2 + pipeline-parallel-size: 1 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + tensor-parallel-size: 1 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + NCCL_CUMEM_ENABLE: '1' + TILELANG_CLEANUP_TEMP_FILES: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_SERVER_DEV_MODE: '1' + args: + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + enable-sleep-mode: true + gpu-memory-utilization: 0.9 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + max-cudagraph-capture-size: 128 + max-model-len: 9280 + max-num-batched-tokens: 128 + max-num-seqs: 128 + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + no-enable-prefix-caching: true + pipeline-parallel-size: 1 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + stream-interval: 50 + tensor-parallel-size: 8 + tokenizer-mode: deepseek_v4 + trust-remote-code: true +benchmark: + concurrencies: "32x128" + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + isl: 8192 + osl: 1024 + req_rate: "inf" + type: "sa-bench" + use_chat_template: true +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.23.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.23.0" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c64-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c64-stp.yaml new file mode 100644 index 0000000000..c9ab89708b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c64-stp.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: "svf-vllm-disagg-b200-low-latency-c64" +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.23.0" + precision: "fp4" +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" +setup_script: vllm-container-deps.sh +slurm: + time_limit: "8:00:00" +health_check: + interval_seconds: 10 + max_attempts: 1440 +sbatch_directives: + segment: "1" +resources: + gpu_type: b200 + gpus_per_node: 8 +frontend: + enable_multiple_frontends: false + type: dynamo +engine: + type: vllm + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids + connector: +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + NCCL_CUMEM_ENABLE: '1' + TILELANG_CLEANUP_TEMP_FILES: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' + VLLM_SERVER_DEV_MODE: '1' + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' + args: + block-size: 256 + data-parallel-rpc-port: 13345 + data-parallel-size: 8 + enable-expert-parallel: true + enable-sleep-mode: true + enforce-eager: true + gpu-memory-utilization: 0.8 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + max-model-len: 9280 + max-num-batched-tokens: 32768 + max-num-seqs: 16 + no-async-scheduling: true + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + no-enable-prefix-caching: true + numa-bind: true + offload-group-size: 3 + offload-num-in-group: 1 + offload-prefetch-step: 2 + pipeline-parallel-size: 1 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + tensor-parallel-size: 1 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + NCCL_CUMEM_ENABLE: '1' + TILELANG_CLEANUP_TEMP_FILES: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_SERVER_DEV_MODE: '1' + args: + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + enable-sleep-mode: true + gpu-memory-utilization: 0.9 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + max-cudagraph-capture-size: 256 + max-model-len: 9280 + max-num-batched-tokens: 256 + max-num-seqs: 256 + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + no-enable-prefix-caching: true + pipeline-parallel-size: 1 + served-model-name: deepseek-ai/DeepSeek-V4-Pro + stream-interval: 50 + tensor-parallel-size: 8 + tokenizer-mode: deepseek_v4 + trust-remote-code: true +benchmark: + concurrencies: "64" + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + isl: 8192 + osl: 1024 + req_rate: "inf" + type: "sa-bench" + use_chat_template: true +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.23.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.23.0" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p2d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p2d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..9a42177af2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p2d-dep8-dep8-mtp.yaml @@ -0,0 +1,153 @@ +schema: 2 +name: "svf-vllm-disagg-b200-1p2d-dep8-dep8-mtp" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +sbatch_directives: + segment: "1" +health_check: + max_attempts: 90 + interval_seconds: 10 +resources: + gpu_type: "b200" + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids + connector: +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NVSHMEM_INFO: "1" + NVSHMEM_DEBUG: "INFO" + NCCL_NVLS_ENABLE: "1" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + # numa-bind: true + tokenizer-mode: deepseek_v4 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NCCL_NVLS_ENABLE: "1" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 128 + max-cudagraph-capture-size: 128 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "128x256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-2p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-2p1d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..0bceada11d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-2p1d-dep8-dep8-mtp.yaml @@ -0,0 +1,152 @@ +schema: 2 +name: "svf-vllm-disagg-b200-2p1d-dep8-dep8-mtp" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +sbatch_directives: + segment: "1" +health_check: + max_attempts: 90 + interval_seconds: 10 +resources: + gpu_type: "b200" + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids + connector: +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 8 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NVSHMEM_INFO: "1" + NVSHMEM_DEBUG: "INFO" + NCCL_NVLS_ENABLE: "1" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + # numa-bind: true + tokenizer-mode: deepseek_v4 + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NCCL_NVLS_ENABLE: "1" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1024" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-3p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-3p1d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..a663e23e8c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b200-fp4/8k1k/disagg-3p1d-dep8-dep8-mtp.yaml @@ -0,0 +1,152 @@ +schema: 2 +name: "svf-vllm-disagg-b200-3p1d-dep8-dep8-mtp" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +health_check: + max_attempts: 90 + interval_seconds: 10 +sbatch_directives: + segment: "1" +resources: + gpu_type: "b200" + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids + connector: +roles: + prefill: + nodes: 3 + workers: 3 + gpus: 8 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NVSHMEM_INFO: "1" + NVSHMEM_DEBUG: "INFO" + NCCL_NVLS_ENABLE: "1" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + speculative-config: '{"method":"mtp","num_speculative_tokens":1}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + # numa-bind: true + tokenizer-mode: deepseek_v4 + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 + TORCH_SYMMMEM: "NVSHMEM" + NVSHMEM_DISABLE_CUDA_VMM: "1" + NVSHMEM_SYMMETRIC_SIZE: "16g" + NCCL_NVLS_ENABLE: "1" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-expert-parallel: true + enable-ep-weight-filter: true + speculative-config: '{"method":"mtp","num_speculative_tokens":1}' + attention-config: '{"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b300-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c256x1024-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b300-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c256x1024-stp.yaml new file mode 100644 index 0000000000..e7c3f0b7cb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b300-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c256x1024-stp.yaml @@ -0,0 +1,153 @@ +schema: 2 +name: "svf-vllm-disagg-b300-mid-curve-megamoe" + +# B300 adaptation of the DSV4 GB200/B200 vLLM disagg recipe. Each worker uses +# one full 8-GPU B300 node, plus a dedicated NATS/etcd infra node. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.23.0" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "b300" + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids + connector: +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + attention-config: '{"use_fp4_indexer_cache": true}' + enforce-eager: true + max-model-len: 9280 + max-num-seqs: 16 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 32768 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + numa-bind: true + tokenizer-mode: deepseek_v4 + reasoning-parser: deepseek_v4 + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + attention-config: '{"use_fp4_indexer_cache": true}' + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + gpu-memory-utilization: 0.85 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + reasoning-parser: deepseek_v4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "256x1024" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.23.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.23.0" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b300-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c1x32x64x128-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b300-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c1x32x64x128-stp.yaml new file mode 100644 index 0000000000..7d69a1e796 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b300-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c1x32x64x128-stp.yaml @@ -0,0 +1,147 @@ +schema: 2 +name: "svf-vllm-disagg-b300-low-latency" + +# B300 adaptation of the DSV4 GB200/B200 vLLM disagg recipe. Each worker uses +# one full 8-GPU B300 node, plus a dedicated NATS/etcd infra node. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.23.0" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "b300" + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids + connector: +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + attention-config: '{"use_fp4_indexer_cache": true}' + enforce-eager: true + max-model-len: 16384 + max-num-seqs: 16 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 32768 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + gpu-memory-utilization: 0.8 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + numa-bind: true + offload-group-size: 3 + offload-num-in-group: 1 + offload-prefetch-step: 2 + tokenizer-mode: deepseek_v4 + reasoning-parser: deepseek_v4 + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + max-model-len: 16384 + max-num-seqs: 256 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + reasoning-parser: deepseek_v4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x32x64x128" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + container: + image: "vllm/vllm-openai:v0.23.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.23.0" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b300-fp4/8k1k/disagg-high-tpt-megamoe-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b300-fp4/8k1k/disagg-high-tpt-megamoe-stp.yaml new file mode 100644 index 0000000000..b718aa2aaa --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/b300-fp4/8k1k/disagg-high-tpt-megamoe-stp.yaml @@ -0,0 +1,153 @@ +schema: 2 +name: "svf-vllm-disagg-b300-high-tpt-megamoe" + +# B300 adaptation of the DSV4 GB200/B200 vLLM disagg recipe. Each worker uses +# one full 8-GPU B300 node, plus a dedicated NATS/etcd infra node. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.23.0" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "b300" + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids + connector: +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 8 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + attention-config: '{"use_fp4_indexer_cache": true}' + enforce-eager: true + max-model-len: 9280 + max-num-seqs: 16 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 32768 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + numa-bind: true + tokenizer-mode: deepseek_v4 + reasoning-parser: deepseek_v4 + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + NCCL_CUMEM_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + attention-config: '{"use_fp4_indexer_cache": true}' + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + gpu-memory-utilization: 0.85 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + reasoning-parser: deepseek_v4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.23.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.23.0" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/agg-tp8-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/agg-tp8-c1-mtp.yaml new file mode 100644 index 0000000000..d0292ed988 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/agg-tp8-c1-mtp.yaml @@ -0,0 +1,89 @@ +schema: 2 +name: "svf-vllm-agg-gb200-low-latency-mtp2" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.20.1-ubuntu2404" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + attention-config: '{"use_fp4_indexer_cache":true}' + tokenizer-mode: deepseek_v4 + max-model-len: 9280 + max-num-seqs: 4 + max-num-batched-tokens: 8192 + max-cudagraph-capture-size: 4 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.9 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.20.1-ubuntu2404" + frameworks: + dynamo: "1.2.0.dev20260426" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c128x256x512x1024-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c128x256x512x1024-mtp.yaml new file mode 100644 index 0000000000..32a7afa9c5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c128x256x512x1024-mtp.yaml @@ -0,0 +1,155 @@ +schema: 2 +name: "svf-vllm-disagg-gb200-mid-curve-megamoe-mtp2" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.21.0-ubuntu2404" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-hybrid-lb: true + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + enforce-eager: true + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + attention-config: '{"use_fp4_indexer_cache":true}' + max-model-len: 9280 + max-num-seqs: 8 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + numa-bind: true + tokenizer-mode: deepseek_v4 + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-hybrid-lb: true + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + attention-config: '{"use_fp4_indexer_cache":true}' + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "128x256x512x1024" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.21.0-ubuntu2404" + frameworks: + dynamo: "1.2.0.dev20260426" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c256x512x1024-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c256x512x1024-stp.yaml new file mode 100644 index 0000000000..b194b7f609 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c256x512x1024-stp.yaml @@ -0,0 +1,167 @@ +schema: 2 +name: "svf-vllm-disagg-gb200-mid-curve-megamoe" + +# Mirrored from NVIDIA/srt-slurm aflowers/vllm-gb200-v0.20.0 branch: +# recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-mid-curve-megamoe.yaml +# +# Topology: 1 prefill (DEP=8) + 1 decode (DEP=8). 5 nodes total with a +# dedicated NATS/etcd infra node. MegaMOE mid-curve points at concurrency +# 256/512/1024 with no CPU/NVMe offload. +# +# Local deltas vs upstream: +# * model.path uses the deepseek-v4-pro-mxfp4 alias from +# runners/launch_gb200-nv.sh. +# * model.container set to vllm/vllm-openai:v0.25.1 to +# match nvidia-master.yaml image (which the launch script registers as +# the alias key in srtslurm.yaml). Upstream variants ship either the +# non-dynamo floating tag or a sha256 pin. +# * slurm.time_limit + health_check set to 8h / 1440 attempts to +# absorb cold-cache /mnt/numa1 model loads. +model: + path: "deepseek-v4-pro-mxfp4" + container: "vllm/vllm-openai:v0.25.1" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated +frontend: + type: dynamo + enable_multiple_frontends: true + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" +engine: + type: vllm + connector: +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + enforce-eager: true + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 32768 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + numa-bind: true + tokenizer-mode: deepseek_v4 + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "256x512x1024" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "b5968e9190ef611bbf34a7229255be88a0e937c1" + container: + image: "vllm/vllm-openai:v0.25.1" + frameworks: + dynamo: "1.3.0.dev1" + vllm: "0.25.1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c1-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c1-stp.yaml new file mode 100644 index 0000000000..5d8662dbe0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c1-stp.yaml @@ -0,0 +1,164 @@ +schema: 2 +name: "svf-vllm-disagg-gb200-low-latency" + +# Mirrored from NVIDIA/srt-slurm aflowers/vllm-gb200-v0.20.0 branch: +# recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-low-latency.yaml +# +# Topology: 1 prefill (DEP=8) + 1 decode (TP=8). 5 nodes total with a +# dedicated NATS/etcd infra node. Single-concurrency point for low latency. +# +# Local deltas vs upstream: +# * model.path uses the deepseek-v4-pro-mxfp4 alias from +# runners/launch_gb200-nv.sh. +# * model.container set to vllm/vllm-openai:v0.25.1 to +# match nvidia-master.yaml image (which the launch script registers as +# the alias key in srtslurm.yaml). Upstream variants ship either the +# non-dynamo floating tag or a sha256 pin. +# * slurm.time_limit + health_check set to 8h / 1440 attempts to +# absorb cold-cache /mnt/numa1 model loads. +model: + path: "deepseek-v4-pro-mxfp4" + container: "vllm/vllm-openai:v0.25.1" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated +frontend: + type: dynamo + enable_multiple_frontends: true + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" +engine: + type: vllm + connector: +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enforce-eager: true + max-model-len: 16384 + max-num-seqs: 16 + max-num-batched-tokens: 32768 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.8 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + numa-bind: true + offload-group-size: 3 + offload-num-in-group: 1 + offload-prefetch-step: 2 + # offload-params: "w13_weight w2_weight w13_weight_scale w2_weight_scale wq_b wo_a wo_b shared_experts" + tokenizer-mode: deepseek_v4 + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 +# data-parallel-size: 8 +# data-parallel-rpc-port: 13345 +# enable-expert-parallel: true + max-model-len: 16384 + max-num-seqs: 256 + max-cudagraph-capture-size: 256 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "b5968e9190ef611bbf34a7229255be88a0e937c1" + container: + image: "vllm/vllm-openai:v0.25.1" + frameworks: + dynamo: "1.3.0.dev1" + vllm: "0.25.1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p4d-p-tp1-dp8-d-tp8-c16x32x64-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p4d-p-tp1-dp8-d-tp8-c16x32x64-mtp.yaml new file mode 100644 index 0000000000..79eb75d38a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p4d-p-tp1-dp8-d-tp8-c16x32x64-mtp.yaml @@ -0,0 +1,142 @@ +schema: 2 +name: svf-vllm-disagg-gb200-low-latency-mtp2 + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.20.1-ubuntu2404" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb200 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + connector: +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + TILELANG_CLEANUP_TEMP_FILES: '1' + VLLM_USE_NCCL_SYMM_MEM: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_TLS: cuda_copy,cuda_ipc,tcp + UCX_CUDA_IPC_ENABLE_MNNVL: y + NCCL_P2P_LEVEL: NVL + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-hybrid-lb: true + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enforce-eager: true + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + attention-config: '{"use_fp4_indexer_cache":true}' + max-model-len: 9280 + max-num-seqs: 8 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + numa-bind: true + tokenizer-mode: deepseek_v4 + decode: + nodes: 8 + workers: 4 + gpus: 8 + env: + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + TILELANG_CLEANUP_TEMP_FILES: '1' + VLLM_USE_NCCL_SYMM_MEM: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_TLS: cuda_copy,cuda_ipc,tcp + UCX_CUDA_IPC_ENABLE_MNNVL: y + NCCL_P2P_LEVEL: NVL + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + attention-config: '{"use_fp4_indexer_cache":true}' + max-model-len: 9280 + max-num-seqs: 256 + max-cudagraph-capture-size: 256 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: 16x32x64 + req_rate: inf + use_chat_template: true + custom_tokenizer: sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.20.1-ubuntu2404" + frameworks: + dynamo: "1.2.0.dev20260426" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p4d-p-tp1-dp8-d-tp8-c256x512-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p4d-p-tp1-dp8-d-tp8-c256x512-stp.yaml new file mode 100644 index 0000000000..cbbe7d4546 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p4d-p-tp1-dp8-d-tp8-c256x512-stp.yaml @@ -0,0 +1,166 @@ +schema: 2 +name: "svf-vllm-disagg-gb200-low-middle-curve" + +# Mirrored from NVIDIA/srt-slurm aflowers/vllm-gb200-v0.20.0 branch: +# recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-low-middle-curve.yaml +# +# Topology: 1 prefill (DEP=8) + 4 decode (TP=8). 11 nodes total with a +# dedicated NATS/etcd infra node. Low-middle curve points at concurrencies +# 256 and 512. +# +# Local deltas vs upstream: +# * model.path uses the deepseek-v4-pro-mxfp4 alias from +# runners/launch_gb200-nv.sh. +# * model.container set to vllm/vllm-openai:v0.25.1 to +# match nvidia-master.yaml image (which the launch script registers as +# the alias key in srtslurm.yaml). Upstream variants ship either the +# non-dynamo floating tag or a sha256 pin. +# * slurm.time_limit + health_check set to 8h / 1440 attempts to +# absorb cold-cache /mnt/numa1 model loads. +model: + path: "deepseek-v4-pro-mxfp4" + container: "vllm/vllm-openai:v0.25.1" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated +frontend: + type: dynamo + enable_multiple_frontends: true + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" +engine: + type: vllm + connector: +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enforce-eager: true + max-model-len: 16384 + max-num-seqs: 16 + max-num-batched-tokens: 32768 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.8 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + numa-bind: true + offload-group-size: 3 + offload-num-in-group: 1 + offload-prefetch-step: 2 + # offload-params: "w13_weight w2_weight w13_weight_scale w2_weight_scale wq_b wo_a wo_b shared_experts" + tokenizer-mode: deepseek_v4 + decode: + nodes: 8 + workers: 4 + gpus: 8 + + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 +# data-parallel-size: 8 +# data-parallel-rpc-port: 13345 +# enable-expert-parallel: true + max-model-len: 16384 + max-num-seqs: 256 + max-cudagraph-capture-size: 256 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.85 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "256x512" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "b5968e9190ef611bbf34a7229255be88a0e937c1" + container: + image: "vllm/vllm-openai:v0.25.1" + frameworks: + dynamo: "1.3.0.dev1" + vllm: "0.25.1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-3p1d-p-tp1-dp8-d-tp1-dp8-c4096-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-3p1d-p-tp1-dp8-d-tp1-dp8-c4096-stp.yaml new file mode 100644 index 0000000000..61f20c13c0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-3p1d-p-tp1-dp8-d-tp1-dp8-c4096-stp.yaml @@ -0,0 +1,167 @@ +schema: 2 +name: "svf-vllm-disagg-gb200-max-tpt-megamoe" + +# Mirrored from NVIDIA/srt-slurm aflowers/vllm-gb200-v0.20.0 branch: +# recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-max-tpt-megamoe.yaml +# +# Topology: 3 prefill (DEP=8 each) + 1 decode (DEP=8). 9 nodes total with a +# dedicated NATS/etcd infra node. MegaMOE max-throughput point at concurrency +# 4096 with no CPU/NVMe offload. +# +# Local deltas vs upstream: +# * model.path uses the deepseek-v4-pro-mxfp4 alias from +# runners/launch_gb200-nv.sh. +# * model.container set to vllm/vllm-openai:v0.25.1 to +# match nvidia-master.yaml image (which the launch script registers as +# the alias key in srtslurm.yaml). Upstream variants ship either the +# non-dynamo floating tag or a sha256 pin. +# * slurm.time_limit + health_check set to 8h / 1440 attempts to +# absorb cold-cache /mnt/numa1 model loads. +model: + path: "deepseek-v4-pro-mxfp4" + container: "vllm/vllm-openai:v0.25.1" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated +frontend: + type: dynamo + enable_multiple_frontends: true + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" +engine: + type: vllm + connector: +roles: + prefill: + nodes: 6 + workers: 3 + gpus: 8 + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + enforce-eager: true + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 32768 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + numa-bind: true + tokenizer-mode: deepseek_v4 + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "b5968e9190ef611bbf34a7229255be88a0e937c1" + container: + image: "vllm/vllm-openai:v0.25.1" + frameworks: + dynamo: "1.3.0.dev1" + vllm: "0.25.1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-high-tpt-megamoe-mtp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-high-tpt-megamoe-mtp2.yaml new file mode 100644 index 0000000000..50067a7698 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-high-tpt-megamoe-mtp2.yaml @@ -0,0 +1,155 @@ +schema: 2 +name: "svf-vllm-disagg-gb200-high-tpt-megamoe-mtp2" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.20.1-ubuntu2404" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + connector: +roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + enforce-eager: true + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + attention-config: '{"use_fp4_indexer_cache":true}' + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 32768 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.94 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + numa-bind: true + tokenizer-mode: deepseek_v4 + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + attention-config: '{"use_fp4_indexer_cache":true}' + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 1024 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1024" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.20.1-ubuntu2404" + frameworks: + dynamo: "1.2.0.dev20260426" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-high-tpt-megamoe-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-high-tpt-megamoe-stp.yaml new file mode 100644 index 0000000000..f77c4558dd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-high-tpt-megamoe-stp.yaml @@ -0,0 +1,167 @@ +schema: 2 +name: "svf-vllm-disagg-gb200-high-tpt-megamoe" + +# Mirrored from NVIDIA/srt-slurm aflowers/vllm-gb200-v0.20.0 branch: +# recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-high-tpt-megamoe.yaml +# +# Topology: 2 prefill (DEP=8 each) + 1 decode (DEP=8). 7 nodes total with a +# dedicated NATS/etcd infra node. MegaMOE high-throughput point at concurrency +# 4096 with no CPU/NVMe offload. +# +# Local deltas vs upstream: +# * model.path uses the deepseek-v4-pro-mxfp4 alias from +# runners/launch_gb200-nv.sh. +# * model.container set to vllm/vllm-openai:v0.25.1 to +# match nvidia-master.yaml image (which the launch script registers as +# the alias key in srtslurm.yaml). Upstream variants ship either the +# non-dynamo floating tag or a sha256 pin. +# * slurm.time_limit + health_check set to 8h / 1440 attempts to +# absorb cold-cache /mnt/numa1 model loads. +model: + path: "deepseek-v4-pro-mxfp4" + container: "vllm/vllm-openai:v0.25.1" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated +frontend: + type: dynamo + enable_multiple_frontends: true + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" +engine: + type: vllm + connector: +roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + enforce-eager: true + max-model-len: 9280 + max-num-seqs: 16 + max-num-batched-tokens: 32768 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + numa-bind: true + tokenizer-mode: deepseek_v4 + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + DYN_REQUEST_PLANE_CODEC: "msgpack" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_P2P_LEVEL: NVL + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + max-model-len: 9280 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "b5968e9190ef611bbf34a7229255be88a0e937c1" + container: + image: "vllm/vllm-openai:v0.25.1" + frameworks: + dynamo: "1.3.0.dev1" + vllm: "0.25.1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml new file mode 100644 index 0000000000..9c604d726b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml @@ -0,0 +1,112 @@ +schema: 2 +name: dsv4-gb200-vllm-agentic-mtp-agg-dep8 + +model: + path: deepseek-v4-pro + container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + precision: fp4 + +identity: + model: {repo: deepseek-ai/DeepSeek-V4-Pro} + container: {image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106} + frameworks: {dynamo: "1.3.1"} + +dynamo: + install: true + + source: + pypi: "1.3.1" +setup_script: vllm-container-deps.sh +environment: {ETCD_LEASE_TTL: "7200"} +slurm: {time_limit: "8:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} + +resources: + gpu_type: gb200 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: kv + router-reset-states: true + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 256 + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + + env: + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 8192 + trust-remote-code: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + gpu-memory-utilization: 0.90 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + kv-cache-metrics: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml new file mode 100644 index 0000000000..71f279e59b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml @@ -0,0 +1,160 @@ +schema: 2 +name: "svf-vllm-agg-gb200-tp8-c4-mtp2-agentic" + +# GB200 AgentX aggregate topology: one TP8 worker spans two four-GPU +# nodes and serves both prefill and decode at concurrency 4. Keep at least +# 16 sequence slots and otherwise size the scheduler at 4x concurrency. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.2.1" + +dynamo: + install: true + + source: + wheel: "1.2.1" +environment: + DYNAMO_WHEEL_DIRS: "/srtctl-wheels" + # The frontend shares Grace CPU capacity with the long TP8 cold start. + ETCD_LEASE_TTL: "7200" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +engine: + type: vllm + connector: +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-agg-mtp2-{job_id}" + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 16 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + gpu-memory-utilization: 0.94 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep aggregate workers in the multinode result schema so ingestion uses + # the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml new file mode 100644 index 0000000000..58e999b60d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml @@ -0,0 +1,159 @@ +schema: 2 +name: "svf-vllm-agg-gb200-tp8-c8-mtp2-agentic" + +# GB200 AgentX aggregate topology: one TP8 worker spans two four-GPU +# nodes and serves both prefill and decode at concurrency 8. Size max-num-seqs at +# 4x concurrency and expand the MTP CUDA-graph envelope to match. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.2.1" + +dynamo: + install: true + + source: + wheel: "1.2.1" +environment: + DYNAMO_WHEEL_DIRS: "/srtctl-wheels" + ETCD_LEASE_TTL: "7200" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +engine: + type: vllm + connector: +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-agg-tp8-c8-mtp2-{job_id}" + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + gpu-memory-utilization: 0.94 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml new file mode 100644 index 0000000000..845cbf966f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml @@ -0,0 +1,122 @@ +schema: 2 +name: dsv4-gb200-vllm-agentic-mtp-agg-tp8 + +model: + path: deepseek-v4-pro + container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + precision: fp4 + +identity: + model: + repo: deepseek-ai/DeepSeek-V4-Pro + container: + image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + frameworks: + dynamo: "1.3.1" + +dynamo: + install: true + + source: + pypi: "1.3.1" +setup_script: vllm-container-deps.sh + +environment: + ETCD_LEASE_TTL: "7200" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: gb200 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: kv + router-reset-states: true + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 256 + +engine: + type: vllm + connector: +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + + env: + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + gpu-memory-utilization: 0.90 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + kv-cache-metrics: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml new file mode 100644 index 0000000000..f9c62a6a24 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml @@ -0,0 +1,221 @@ +schema: 2 +name: "svf-vllm-disagg-gb200-1p1d-dep8-dep8-c128-mtp2-agentic" + +# GB200 AgentX MTP3 topology: one DEP8 prefill worker feeds one +# DEP8 decode worker at concurrency 128. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.3.0.dev20260720" + +dynamo: + install: true + + source: + wheel: "1.3.0.dev20260720" +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "140GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c128-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c128-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 512 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml new file mode 100644 index 0000000000..01c6e6e5af --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml @@ -0,0 +1,221 @@ +schema: 2 +name: "svf-vllm-disagg-gb200-1p1d-dep8-dep8-c256-mtp2-agentic" + +# GB200 AgentX MTP3 topology: one DEP8 prefill worker feeds one +# DEP8 decode worker at concurrency 256. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.3.0.dev20260720" + +dynamo: + install: true + + source: + wheel: "1.3.0.dev20260720" +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "140GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c256-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.85 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c256-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 512 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..178f6af471 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml @@ -0,0 +1,133 @@ +schema: 2 +name: dsv4-gb200-vllm-agentic-mtp-disagg-1p1d-dep8-dep8 + +model: + path: deepseek-v4-pro + container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + precision: fp4 + +identity: + model: {repo: deepseek-ai/DeepSeek-V4-Pro} + container: {image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106} + frameworks: {dynamo: "1.3.1"} + +dynamo: + install: true + + source: + pypi: "1.3.1" +setup_script: vllm-container-deps.sh +environment: {ETCD_LEASE_TTL: "7200"} +slurm: {time_limit: "8:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} + +resources: + gpu_type: gb200 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + # Long AgentX prefills can exceed Dynamo's request-plane default while + # the healthy DEP8 worker is still computing the first response. + DYN_TCP_REQUEST_TIMEOUT: "60" + args: + router-mode: kv + router-reset-states: true + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 256 + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: &worker_environment + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: &dep8_config + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + enforce-eager: true + block-size: 256 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + kv-cache-metrics: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: *worker_environment + args: + <<: *dep8_config + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' + enforce-eager: false + gpu-memory-utilization: 0.90 + max-num-seqs: 1024 + max-num-batched-tokens: 1024 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 1024 + stream-interval: 10 + +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml new file mode 100644 index 0000000000..0bc695d779 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml @@ -0,0 +1,223 @@ +schema: 2 +name: "svf-vllm-disagg-gb200-2p1d-dep8-dep12-c576-mtp2-agentic" + +# GB200 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP12 decode worker at concurrency 576. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.3.0.dev20260720" + +dynamo: + install: true + + source: + wheel: "1.3.0.dev20260720" +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "140GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-prefill-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + nodes: 3 + workers: 1 + gpus: 12 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-dep12-decode-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 12 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml new file mode 100644 index 0000000000..9ead5965b0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml @@ -0,0 +1,223 @@ +schema: 2 +name: "svf-vllm-disagg-gb200-2p1d-dep8-dep16-c512-mtp2-agentic" + +# GB200 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP16 decode worker at concurrency 512. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.3.0.dev20260720" + +dynamo: + install: true + + source: + wheel: "1.3.0.dev20260720" +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "140GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-prefill-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-dep16-decode-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..fb540cdbf5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml @@ -0,0 +1,130 @@ +schema: 2 +name: dsv4-gb200-vllm-agentic-mtp-disagg-2p1d-dep8-dep8 + +model: + path: deepseek-v4-pro + container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + precision: fp4 + +identity: + model: {repo: deepseek-ai/DeepSeek-V4-Pro} + container: {image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106} + frameworks: {dynamo: "1.3.1"} + +dynamo: {install: true, source: {pypi: "1.3.1"}} +setup_script: vllm-container-deps.sh +environment: {ETCD_LEASE_TTL: "7200"} +slurm: {time_limit: "8:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} + +resources: + gpu_type: gb200 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + # Long AgentX prefills can exceed Dynamo's request-plane default while + # the healthy DEP8 workers are still computing their first responses. + DYN_TCP_REQUEST_TIMEOUT: "60" + args: + router-mode: kv + router-reset-states: true + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 256 + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + env: &worker_environment + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: &dep8_config + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + enforce-eager: true + block-size: 256 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + kv-cache-metrics: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: *worker_environment + args: + <<: *dep8_config + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' + enforce-eager: false + gpu-memory-utilization: 0.90 + max-num-seqs: 1024 + max-num-batched-tokens: 1024 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 1024 + stream-interval: 10 + +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-1p6d-dep4-tp4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-1p6d-dep4-tp4-stp.yaml new file mode 100644 index 0000000000..c1479614dc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-1p6d-dep4-tp4-stp.yaml @@ -0,0 +1,126 @@ +schema: 2 +name: "svf-vllm-disagg-gb300-1p6d-dep4-tp4" + +# Topology: 1 prefill (DEP=4) + 6 decode (TP=4). 7 GB300 nodes (1P + 6D = 28 +# GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra node. Low-mid curve +# point at concurrency 192. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + attention-config: '{"use_fp4_indexer_cache": true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + enforce-eager: true + max-model-len: 16384 + max-num-seqs: 256 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.9 + enable-ep-weight-filter: true + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + decode: + nodes: 6 + workers: 6 + gpus: 4 + + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 16384 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-ep-weight-filter: true + all2all-backend: "flashinfer_nvlink_one_sided" + no-enable-flashinfer-autotune: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "192" + req_rate: "inf" + custom_tokenizer: sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-1p9d-tep4-tp4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-1p9d-tep4-tp4-stp.yaml new file mode 100644 index 0000000000..26bcfcd1fd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-1p9d-tep4-tp4-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "svf-vllm-disagg-gb300-1p9d-tep4-tp4" + +# Topology: 1 prefill (TEP=4) + 9 decode (TP=4). 10 GB300 nodes (1P + 9D = 40 +# GPUs at 4 GPUs/node), NATS/etcd colocated on the prefill node. +# Wide-decode point at concurrency 18 — each decode worker holds a +# single replica. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: true + enforce-eager: true + max-model-len: 16384 + max-num-seqs: 256 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.9 + enable-ep-weight-filter: true + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + decode: + nodes: 9 + workers: 9 + gpus: 4 + + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 16384 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-ep-weight-filter: true + all2all-backend: "flashinfer_nvlink_one_sided" + no-enable-flashinfer-autotune: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "18" + req_rate: "inf" + custom_tokenizer: sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep8-24-c4096-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep8-24-c4096-stp.yaml new file mode 100644 index 0000000000..0d3e29365e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep8-24-c4096-stp.yaml @@ -0,0 +1,135 @@ +schema: 2 +name: "svf-vllm-disagg-gb300-4p1d-dep4-dep8-24" + +# Topology: 4 prefill (DEP=4 each) + 1 decode (DEP=8). 6 GB300 nodes (4P + 2D +# = 24 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra node. +# Max-throughput point at concurrency 4096 with deep_gemm_amxf4_mega_moe on +# both workers. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" + precision: "fp4" + +dynamo: + install: true + + source: + wheel: "1.2.0.dev20260426" +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 4 + workers: 4 + gpus: 4 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enforce-eager: true + max-model-len: 16384 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + safetensors-load-strategy: "prefetch" + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + no-async-scheduling: true + tokenizer-mode: deepseek_v4 + enable-ep-weight-filter: true + enable-sleep-mode: true + moe-backend: "deep_gemm_amxf4_mega_moe" + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 16384 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + enable-ep-weight-filter: true + enable-sleep-mode: true + moe-backend: "deep_gemm_amxf4_mega_moe" + no-enable-flashinfer-autotune: true + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" + custom_tokenizer: sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-5p1d-dep4-dep8-28-c4096-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-5p1d-dep4-dep8-28-c4096-stp.yaml new file mode 100644 index 0000000000..d1a15c34f7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-5p1d-dep4-dep8-28-c4096-stp.yaml @@ -0,0 +1,135 @@ +schema: 2 +name: "svf-vllm-disagg-gb300-5p1d-dep4-dep8-28" + +# Topology: 5 prefill (DEP=4 each) + 1 decode (DEP=8). 7 GB300 nodes (5P + 2D +# = 28 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra node. +# Max-throughput point at concurrency 4096 with deep_gemm_amxf4_mega_moe on +# both workers. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" + precision: "fp4" + +dynamo: + install: true + + source: + wheel: "1.2.0.dev20260426" +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 5 + workers: 5 + gpus: 4 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enforce-eager: true + max-model-len: 16384 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + safetensors-load-strategy: "prefetch" + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + no-async-scheduling: true + tokenizer-mode: deepseek_v4 + enable-ep-weight-filter: true + enable-sleep-mode: true + moe-backend: "deep_gemm_amxf4_mega_moe" + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 16384 + max-num-seqs: 384 + max-cudagraph-capture-size: 512 + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + enable-ep-weight-filter: true + enable-sleep-mode: true + moe-backend: "deep_gemm_amxf4_mega_moe" + no-enable-flashinfer-autotune: true + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" + custom_tokenizer: sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-6p1d-dep4-dep8-32-c4096-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-6p1d-dep4-dep8-32-c4096-stp.yaml new file mode 100644 index 0000000000..e32c4191cd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-6p1d-dep4-dep8-32-c4096-stp.yaml @@ -0,0 +1,135 @@ +schema: 2 +name: "svf-vllm-disagg-gb300-6p1d-dep4-dep8-32" + +# Topology: 6 prefill (DEP=4 each) + 1 decode (DEP=8). 8 GB300 nodes (6P + 2D +# = 32 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra node. +# Max-throughput point at concurrency 4096 with deep_gemm_amxf4_mega_moe on +# both workers. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" + precision: "fp4" + +dynamo: + install: true + + source: + wheel: "1.2.0.dev20260426" +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 6 + workers: 6 + gpus: 4 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enforce-eager: true + max-model-len: 16384 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + safetensors-load-strategy: "prefetch" + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + no-async-scheduling: true + tokenizer-mode: deepseek_v4 + enable-ep-weight-filter: true + enable-sleep-mode: true + moe-backend: "deep_gemm_amxf4_mega_moe" + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 16384 + max-num-seqs: 384 + max-cudagraph-capture-size: 512 + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + enable-ep-weight-filter: true + enable-sleep-mode: true + moe-backend: "deep_gemm_amxf4_mega_moe" + no-enable-flashinfer-autotune: true + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" + custom_tokenizer: sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-7p2d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-7p2d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..55a72bad2b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-7p2d-dep4-dep16-stp.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: "svf-vllm-disagg-gb300-7p2d-dep4-dep16" + +# Topology: 7 prefill (DEP=4) + 2 decode (DEP=16). 15 GB300 nodes (7P + 8D +# = 60 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra node. +# Wide-EP decode max-throughput point at concurrency 3072. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + prefill: + nodes: 7 + workers: 7 + gpus: 4 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + attention-config: '{"use_fp4_indexer_cache": true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + enforce-eager: true + max-model-len: 16384 + max-num-seqs: 256 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + no-async-scheduling: true + block-size: 256 + gpu-memory-utilization: 0.9 + enable-ep-weight-filter: true + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + decode: + nodes: 8 + workers: 2 + gpus: 16 + + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + attention-config: '{"use_fp4_indexer_cache": true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + max-model-len: 16384 + max-num-seqs: 512 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + no-disable-hybrid-kv-cache-manager: true + enable-ep-weight-filter: true + all2all-backend: "flashinfer_nvlink_one_sided" + no-enable-flashinfer-autotune: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "3072" + req_rate: "inf" + custom_tokenizer: sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml new file mode 100644 index 0000000000..dda2443295 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml @@ -0,0 +1,159 @@ +schema: 2 +name: "svf-vllm-agg-gb300-tp4-mtp-agentic" + +# GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node +# and serves both prefill and decode at concurrency 8. Size max-num-seqs at +# 4x concurrency and expand the MTP CUDA-graph envelope to match. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.2.1" + +dynamo: + install: true + + source: + wheel: "1.2.1" +environment: + DYNAMO_WHEEL_DIRS: "/srtctl-wheels" + ETCD_LEASE_TTL: "7200" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +engine: + type: vllm + connector: +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp4-mtp-{job_id}" + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + gpu-memory-utilization: 0.94 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml new file mode 100644 index 0000000000..f17faead2a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml @@ -0,0 +1,160 @@ +schema: 2 +name: "svf-vllm-agg-gb300-tp8-mtp-agentic" + +# GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU +# nodes and serves both prefill and decode at concurrency 4. Keep at least +# 16 sequence slots and otherwise size the scheduler at 4x concurrency. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.2.1" + +dynamo: + install: true + + source: + wheel: "1.2.1" +environment: + DYNAMO_WHEEL_DIRS: "/srtctl-wheels" + # The frontend shares Grace CPU capacity with the long TP8 cold start. + ETCD_LEASE_TTL: "7200" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +engine: + type: vllm + connector: +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-mtp-{job_id}" + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 16 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + gpu-memory-utilization: 0.94 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep aggregate workers in the multinode result schema so ingestion uses + # the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml new file mode 100644 index 0000000000..acdca7c858 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml @@ -0,0 +1,228 @@ +# Derived from PR #2302's DEP8/DEP32 c256 recipe. +# Halve the prefill/decode topology while preserving its conservative +# 4/16/16 decode limits and isolated JIT cache paths. +schema: 2 +name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c128-mtp-agentic" + +# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one +# DEP16 decode worker at concurrency 128. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.4.0" + +dynamo: + install: true + + source: + wheel: "1.4.0" +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c128-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c128-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 4 + max-num-batched-tokens: 16 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 16 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml new file mode 100644 index 0000000000..bb98740ab7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml @@ -0,0 +1,228 @@ +# Derived from PR #2302's DEP8/DEP32 c256 recipe. +# Halve the prefill/decode topology while preserving its conservative +# 8/32/32 decode limits and isolated JIT cache paths. +schema: 2 +name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic" + +# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one +# DEP16 decode worker at concurrency 256. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.4.0" + +dynamo: + install: true + + source: + wheel: "1.4.0" +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c256-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c256-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 8 + max-num-batched-tokens: 32 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 32 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml new file mode 100644 index 0000000000..62867c6566 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml @@ -0,0 +1,221 @@ +schema: 2 +name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic" + +# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one +# DEP8 decode worker at concurrency 256. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.3.0.dev20260720" + +dynamo: + install: true + + source: + wheel: "1.3.0.dev20260720" +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 1024 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml new file mode 100644 index 0000000000..d9d280a6c9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml @@ -0,0 +1,228 @@ +# Source: https://github.com/SemiAnalysisAI/InferenceX/blob/0c33d4615792705ed12bfc204e3a54cfa436cf02/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml +# Runtime and performance arguments follow the current DEP8/DEP16 GB300 P/D +# recipes; throughput-only synthetic MTP acceptance is injected at launch. +schema: 2 +name: "svf-vllm-disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic" + +# GB300 AgentX MTP3 topology: one DEP8 prefill worker feeds one +# DEP16 decode worker at concurrency 512. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.4.0" + +dynamo: + install: true + + source: + wheel: "1.4.0" +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c512-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 16384 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep16-decode-c512-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml new file mode 100644 index 0000000000..922a5343a8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml @@ -0,0 +1,234 @@ +# Source: https://github.com/Inferact/srt-slurm-sa/blob/4a870cd5bc333bf8a312fc65ca1ea82cdab9b2df/recipes/vllm/deepseek-v4-pro/GB300/agentic/agentx-v1.0.1/1p3d-pdep4-dtp8-c3-kv-nixl-mtp-flashinfer-ar-lpt512-psi1.yaml +# Adapted from the source topology to a 1P4D concurrency-4 point for the +# InferenceX AgentX harness; eval-only runs continue to verify real MTP output. +schema: 2 +name: "svf-vllm-disagg-gb300-1p4d-dep4-tp8-c4-mtp-agentic" + +# GB300 high-interactivity AgentX MTP3 topology: one DEP4 prefill worker +# feeds four TP8 decode workers at concurrency 4 through NIXL. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.4.0" + +dynamo: + install: true + source: + wheel: "1.4.0" + request_plane: tcp + +environment: + DYNAMO_WHEEL_DIRS: "/srtctl-wheels" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p4d-{job_id}" + # Multi-node TP8 decode spans two GB300 nodes; match the sibling recipes' + # NCCL/UCX fabric settings (MNNVL/NVLS, IB HCAs). + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + +slurm: + time_limit: "08:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "4" + DURATION: "3600" + KV_OFFLOADING: "none" + ETCD_LEASE_TTL: "120" + DYN_ROUTER_QUEUE_THRESHOLD: "None" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: "14400" + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + args: + router-mode: "kv" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + placement: + node: first_decode +engine: + type: vllm + connector: + dp_launch_mode: per_node + # vLLM KV routing needs prefill KV events so Dynamo can select the + # cache-owning PDEP4 rank. +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + + args: + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 512 + prefill-schedule-interval: 1 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + # MTP capture sizes are tokens: 64 seqs * (1 target + 3 drafts). + max-cudagraph-capture-size: 256 + kv_events: true + decode: + nodes: 8 + workers: 4 + gpus: 8 + + args: + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}}' + # TP8 spans two GB300 nodes; custom all-reduce is single-node only, so + # decode uses the FlashInfer allreduce path like the agg TP8 recipes. + disable-custom-all-reduce: true + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + max-model-len: 1048576 + max-num-seqs: 16 + max-num-batched-tokens: 64 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + # MTP capture sizes are tokens: 16 seqs * (1 target + 3 drafts). + max-cudagraph-capture-size: 64 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' +setup_script: vllm-container-deps.sh + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + # The AgentX client uses localhost:8000, so colocate it with the Dynamo + # frontend launched on the first decode node. + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + SERVED_MODEL_NAME: "deepseek-ai/DeepSeek-V4-Pro" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "4" + DURATION: "3600" + RUNNER_TYPE: "gb300" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + SPEC_DECODING: "mtp" + DISAGG: "true" + OFFLOADING: "none" + KV_OFFLOADING: "none" + TP: "8" + PREFILL_TP: "1" + PREFILL_NUM_WORKERS: "1" + PREFILL_EP: "4" + DECODE_TP: "8" + DECODE_NUM_WORKERS: "4" + DECODE_EP: "1" + EP_SIZE: "1" + AIPERF_MAX_OSL: "none" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + NUM_DATASET_ENTRIES: "393" + HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" + PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" + placement: + node: first_decode diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml new file mode 100644 index 0000000000..2670488be1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml @@ -0,0 +1,179 @@ +schema: 2 +name: "svf-vllm-disagg-gb300-1p6d-dep4-tp4-agentic" + +# Agentic-coding variant of dsv4/vllm/gb300-fp4/8k1k/disagg-1p6d-dep4-tp4-stp.yaml. +# Topology is identical (1 prefill DEP=4 + 6 decode TP=4, 28 GPUs across 7 +# GB300 nodes + 1 dedicated NATS/etcd infra node) so we can compare against +# the fixed-seq-len 1p6d baseline at the same concurrency point (192). +# +# Divergence vs the 8k1k sibling: +# - benchmark.type: sa-bench -> custom (hands off to agentic_srt.sh) +# - max-model-len: removed (let vLLM derive from model config; agentic +# trajectories blow past any small explicit cap) +# - no-enable-prefix-caching: dropped (prefix caching MUST be on for +# trajectory reuse — entire point of agentic) +# Note: --enable-auto-tool-choice / --tool-call-parser / --reasoning-parser +# are NOT set on the worker. The dynamo-vllm worker entrypoint doesn't +# accept them (different arg parser than `vllm serve`). In disagg, chat +# parsing happens at the dynamo frontend, not at the worker. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.21.0-ubuntu2404" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + attention-config: '{"use_fp4_indexer_cache": true}' + moe-backend: "deep_gemm_mega_moe" + # enforce-eager: true + # max-num-seqs: 256 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.9 + enable-ep-weight-filter: true + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + decode: + nodes: 6 + workers: 6 + gpus: 4 + + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + # max-num-seqs: 512 + trust-remote-code: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + enable-ep-weight-filter: true + all2all-backend: "flashinfer_nvlink_one_sided" + no-enable-flashinfer-autotune: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + +# sbatch + srun resource grants for clusters without per-GPU defaults. +# +# mem=0: allocate all available node memory (~868 GB on CW gb300). Without +# this, sbatch only requests ntasks × DefMemPerCPU = 8 × 4 GB = 32 GB for +# the whole job and worker cgroups OOM-kill mid model load (R7-R11 hit +# this; sacct showed AllocTRES mem=4G per step). +# +# cpus-per-task=72: give each task one CW gb300 NUMA socket (144 cores +# split 2 × 72). Critical for the *infra step* (etcd + nats) which +# srtctl spawns without --gres=gpu — on CW that means DefMemPerCPU +# applies and the step gets 1 CPU by default. With 24 dynamo DP ranks +# all hammering etcd for lease keep-alives, single-CPU etcd can't keep +# up and dies (R12 hit this; etcd reported max-cpu-set=1, leases +# deadline-exceeded, infra SIGKILL'd at 16:35:49). 72 CPUs is plenty +# for both etcd + nats AND for vLLM worker auxiliary threads. +# +# nv gb300 doesn't need this because cluster default DefCpuPerGPU=35 +# auto-allocates 4*35=140 CPUs per GPU-bearing task; cw has no per-GPU +# default. Setting it here is safe on both because the value is ≤ node +# CPU count. +# +# srun_options.mem=0 forces each srun step to use the full node memory +# (without it, srun steps default back to cpus_per_task × DefMemPerCPU). +# Docs: docs/config-reference.md#sbatch_directives + #srun_options. +sbatch_directives: + mem: "0" + cpus-per-task: "72" +srun_options: + mem: "0" + # gb300-nv: pyxis maps the calling user (sa-shared) into the container as + # uid 345200007. dpkg refuses to run without EUID 0 even though + # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the agentic_srt + # apt-get install git step fails. --container-remap-root asks pyxis to + # remap us to uid 0 inside the container. srt-slurm renders empty-string + # values as flag-only srun args (see core/slurm.py:250). + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + # Container-side path of the aiperf mmap dataset cache; the host-side + # mount is wired via launch_gb300-*.sh's srtslurm.yaml default_mounts. + # Without this, aiperf re-tokenizes + re-writes ~65 GB of mmap files + # per dataset on every run. + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + # Persistent HF hub cache (also wired via default_mounts) so the trace + # dataset isn't re-downloaded on every run. Overrides the workflow-level + # HF_HUB_CACHE=/mnt/hf_hub_cache, which doesn't exist on these nodes. + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml new file mode 100644 index 0000000000..2b234884ff --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml @@ -0,0 +1,223 @@ +schema: 2 +name: "svf-vllm-disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic" + +# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP12 decode worker at concurrency 1152. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.3.0.dev20260720" + +dynamo: + install: true + + source: + wheel: "1.3.0.dev20260720" +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + nodes: 3 + workers: 1 + gpus: 12 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep12-decode-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 12 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml new file mode 100644 index 0000000000..7b8b762e6f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml @@ -0,0 +1,223 @@ +schema: 2 +name: "svf-vllm-disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic" + +# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP16 decode worker at concurrency 1024. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.3.0.dev20260720" + +dynamo: + install: true + + source: + wheel: "1.3.0.dev20260720" +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep16-decode-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml new file mode 100644 index 0000000000..20e836b93b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml @@ -0,0 +1,188 @@ +schema: 2 +name: "svf-vllm-disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic" + +# Agentic-coding variant of dsv4/vllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep8-24-c4096-stp.yaml. +# Max-throughput shape: 4 prefill (DEP=4 each) + 1 decode (DEP=8). 6 GB300 +# nodes (4P + 2D = 24 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra +# node. Sized for concurrency 4096 with deep_gemm_mega_moe on both workers. +# +# Divergence vs the 8k1k sibling: +# - benchmark.type: sa-bench -> custom (hands off to agentic_srt.sh) +# - max-model-len: removed (let vLLM derive from model config; agentic +# trajectories blow past any small explicit cap) +# - no-enable-prefix-caching: dropped (prefix caching MUST be on for +# trajectory reuse — entire point of agentic) +# Note: --enable-auto-tool-choice / --tool-call-parser / --reasoning-parser +# are NOT set on the worker. The dynamo-vllm worker entrypoint doesn't +# accept them (different arg parser than `vllm serve`). In disagg, chat +# parsing happens at the dynamo frontend, not at the worker. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.21.0-ubuntu2404" + precision: "fp4" + +dynamo: + install: true + source: + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 4 + workers: 4 + gpus: 4 + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + # enforce-eager: true + # Inherited from fixed-sequence recipes; let vLLM select the scheduler + # sequence limit until this is tuned explicitly for the agentic trace. + # max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-flashinfer-autotune: true + safetensors-load-strategy: "prefetch" + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + enable-ep-weight-filter: true + enable-sleep-mode: true + moe-backend: "deep_gemm_mega_moe" + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + TORCH_SYMMMEM: "NVSHMEM" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + # max-num-seqs: 512 + trust-remote-code: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + gpu-memory-utilization: 0.9 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + enable-ep-weight-filter: true + enable-sleep-mode: true + moe-backend: "deep_gemm_mega_moe" + +# sbatch + srun resource grants for clusters without per-GPU defaults. +# +# mem=0: allocate all available node memory (~868 GB on CW gb300). Without +# this, sbatch only requests ntasks × DefMemPerCPU = 8 × 4 GB = 32 GB for +# the whole job and worker cgroups OOM-kill mid model load (R7-R11 hit +# this; sacct showed AllocTRES mem=4G per step). +# +# cpus-per-task=72: give each task one CW gb300 NUMA socket (144 cores +# split 2 × 72). Critical for the *infra step* (etcd + nats) which +# srtctl spawns without --gres=gpu — on CW that means DefMemPerCPU +# applies and the step gets 1 CPU by default. With 24 dynamo DP ranks +# all hammering etcd for lease keep-alives, single-CPU etcd can't keep +# up and dies (R12 hit this; etcd reported max-cpu-set=1, leases +# deadline-exceeded, infra SIGKILL'd at 16:35:49). 72 CPUs is plenty +# for both etcd + nats AND for vLLM worker auxiliary threads. +# +# nv gb300 doesn't need this because cluster default DefCpuPerGPU=35 +# auto-allocates 4*35=140 CPUs per GPU-bearing task; cw has no per-GPU +# default. Setting it here is safe on both because the value is ≤ node +# CPU count. +# +# srun_options.mem=0 forces each srun step to use the full node memory +# (without it, srun steps default back to cpus_per_task × DefMemPerCPU). +# Docs: docs/config-reference.md#sbatch_directives + #srun_options. +sbatch_directives: + mem: "0" + cpus-per-task: "72" +srun_options: + mem: "0" + # gb300-nv: pyxis maps the calling user (sa-shared) into the container as + # uid 345200007. dpkg refuses to run without EUID 0 even though + # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the agentic_srt + # apt-get install git step fails. --container-remap-root asks pyxis to + # remap us to uid 0 inside the container. srt-slurm renders empty-string + # values as flag-only srun args (see core/slurm.py:250). + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + # Container-side path of the aiperf mmap dataset cache; the host-side + # mount is wired via launch_gb300-*.sh's srtslurm.yaml default_mounts. + # Without this, aiperf re-tokenizes + re-writes ~65 GB of mmap files + # per dataset on every run. + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + # Persistent HF hub cache (also wired via default_mounts) so the trace + # dataset isn't re-downloaded on every run. Overrides the workflow-level + # HF_HUB_CACHE=/mnt/hf_hub_cache, which doesn't exist on these nodes. + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b1-c23-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b1-c23-stp.yaml new file mode 100644 index 0000000000..ac855f2a6a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b1-c23-stp.yaml @@ -0,0 +1,145 @@ +schema: 2 +name: "gb200-fp4-glm5_1k1k_lowlat_3" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 128 + cuda-graph-max-bs: 128 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 2112 + mem-fraction-static: 0.785 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 16 + workers: 16 + gpus: 4 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 2112 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + load-balance-method: "round_robin" + moe-runner-backend: "flashinfer_trtllm" + enable-flashinfer-allreduce-fusion: true + mem-fraction-static: 0.918 + max-running-requests: 1 + cuda-graph-max-bs: 1 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "23" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b16-c290-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b16-c290-stp.yaml new file mode 100644 index 0000000000..845c0a5371 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b16-c290-stp.yaml @@ -0,0 +1,145 @@ +schema: 2 +name: "gb200-fp4-glm5_1k1k_lowlat_4" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 128 + cuda-graph-max-bs: 128 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 2112 + mem-fraction-static: 0.785 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 16 + workers: 16 + gpus: 4 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 2112 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + load-balance-method: "round_robin" + moe-runner-backend: "flashinfer_trtllm" + enable-flashinfer-allreduce-fusion: true + mem-fraction-static: 0.918 + max-running-requests: 16 + cuda-graph-max-bs: 16 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "290" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b4-c73-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b4-c73-stp.yaml new file mode 100644 index 0000000000..45a109c50b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b4-c73-stp.yaml @@ -0,0 +1,145 @@ +schema: 2 +name: "gb200-fp4-glm5_1k1k_lowlat_5" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 128 + cuda-graph-max-bs: 128 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 2112 + mem-fraction-static: 0.785 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 16 + workers: 16 + gpus: 4 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 2112 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + load-balance-method: "round_robin" + moe-runner-backend: "flashinfer_trtllm" + enable-flashinfer-allreduce-fusion: true + mem-fraction-static: 0.918 + max-running-requests: 4 + cuda-graph-max-bs: 4 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "73" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2023-c2115-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2023-c2115-stp.yaml new file mode 100644 index 0000000000..361120927f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2023-c2115-stp.yaml @@ -0,0 +1,150 @@ +schema: 2 +name: "gb200-fp4-glm5_1k1k_lowlat_0" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 128 + cuda-graph-max-bs: 128 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 2112 + mem-fraction-static: 0.785 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 2112 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 0 + ep-dispatch-algorithm: "static" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + deepep-config: "/configs/deepep_config.json" + moe-runner-backend: "flashinfer_cutedsl" + enable-flashinfer-allreduce-fusion: false + mem-fraction-static: 0.866 + max-running-requests: 2023 + cuda-graph-max-bs: 2023 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "2115" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b1024-c1156-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b1024-c1156-stp.yaml new file mode 100644 index 0000000000..f0ea6ec111 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b1024-c1156-stp.yaml @@ -0,0 +1,150 @@ +schema: 2 +name: "gb200-fp4-glm5_1k1k_lowlat_1" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 128 + cuda-graph-max-bs: 128 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 2112 + mem-fraction-static: 0.785 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 2112 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 32 + data-parallel-size: 32 + expert-parallel-size: 32 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 0 + ep-dispatch-algorithm: "static" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + deepep-config: "/configs/deepep_config.json" + moe-runner-backend: "flashinfer_cutedsl" + enable-flashinfer-allreduce-fusion: false + mem-fraction-static: 0.858 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1156" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b512-c556-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b512-c556-stp.yaml new file mode 100644 index 0000000000..51ff82d676 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b512-c556-stp.yaml @@ -0,0 +1,150 @@ +schema: 2 +name: "gb200-fp4-glm5_1k1k_lowlat_2" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 128 + cuda-graph-max-bs: 128 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 2112 + mem-fraction-static: 0.785 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 2112 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 32 + data-parallel-size: 32 + expert-parallel-size: 32 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 0 + ep-dispatch-algorithm: "static" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + deepep-config: "/configs/deepep_config.json" + moe-runner-backend: "flashinfer_cutedsl" + enable-flashinfer-allreduce-fusion: false + mem-fraction-static: 0.858 + max-running-requests: 512 + cuda-graph-max-bs: 512 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "556" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-2p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b2048-c3160-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-2p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b2048-c3160-stp.yaml new file mode 100644 index 0000000000..aabd9d8957 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-2p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b2048-c3160-stp.yaml @@ -0,0 +1,150 @@ +schema: 2 +name: "gb200-fp4-glm5_1k1k_maxtpt_0" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 128 + cuda-graph-max-bs: 128 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 2112 + mem-fraction-static: 0.812 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 2112 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 32 + data-parallel-size: 32 + expert-parallel-size: 32 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 0 + ep-dispatch-algorithm: "static" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + deepep-config: "/configs/deepep_config.json" + moe-runner-backend: "flashinfer_cutedsl" + enable-flashinfer-allreduce-fusion: false + mem-fraction-static: 0.858 + max-running-requests: 2048 + cuda-graph-max-bs: 2048 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "3160" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-10p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b1712-c2674-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-10p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b1712-c2674-stp.yaml new file mode 100644 index 0000000000..174c800681 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-10p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b1712-c2674-stp.yaml @@ -0,0 +1,150 @@ +schema: 2 +name: "gb200-fp4-glm5_8k1k_maxtpt_3" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 10 + workers: 10 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 72 + cuda-graph-max-bs: 72 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 9280 + mem-fraction-static: 0.812 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 9600 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 32 + data-parallel-size: 32 + expert-parallel-size: 32 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 0 + ep-dispatch-algorithm: "static" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + deepep-config: "/configs/deepep_config.json" + moe-runner-backend: "flashinfer_cutedsl" + enable-flashinfer-allreduce-fusion: false + mem-fraction-static: 0.812 + max-running-requests: 1712 + cuda-graph-max-bs: 1712 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2674" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b1-c23-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b1-c23-stp.yaml new file mode 100644 index 0000000000..8fcd70e2fa --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b1-c23-stp.yaml @@ -0,0 +1,145 @@ +schema: 2 +name: "gb200-fp4-glm5_8k1k_lowlat_5" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 72 + cuda-graph-max-bs: 72 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 9280 + mem-fraction-static: 0.785 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 16 + workers: 16 + gpus: 4 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 9600 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + load-balance-method: "round_robin" + moe-runner-backend: "flashinfer_trtllm" + enable-flashinfer-allreduce-fusion: true + mem-fraction-static: 0.919 + max-running-requests: 1 + cuda-graph-max-bs: 1 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "23" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b6-c113-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b6-c113-stp.yaml new file mode 100644 index 0000000000..459b1e928b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b6-c113-stp.yaml @@ -0,0 +1,145 @@ +schema: 2 +name: "gb200-fp4-glm5_8k1k_lowlat_4" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 72 + cuda-graph-max-bs: 72 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 9280 + mem-fraction-static: 0.785 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 16 + workers: 16 + gpus: 4 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 9600 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + load-balance-method: "round_robin" + moe-runner-backend: "flashinfer_trtllm" + enable-flashinfer-allreduce-fusion: true + mem-fraction-static: 0.919 + max-running-requests: 6 + cuda-graph-max-bs: 6 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "113" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p4d-p-tp4-ep4-dp4-d-tp4-b32-c133-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p4d-p-tp4-ep4-dp4-d-tp4-b32-c133-stp.yaml new file mode 100644 index 0000000000..ce52ebc0ee --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p4d-p-tp4-ep4-dp4-d-tp4-b32-c133-stp.yaml @@ -0,0 +1,145 @@ +schema: 2 +name: "gb200-fp4-glm5_8k1k_lowlat_0" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 72 + cuda-graph-max-bs: 72 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 9280 + mem-fraction-static: 0.8 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 4 + workers: 4 + gpus: 4 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 9600 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + load-balance-method: "round_robin" + moe-runner-backend: "flashinfer_trtllm" + enable-flashinfer-allreduce-fusion: true + mem-fraction-static: 0.94 + max-running-requests: 32 + cuda-graph-max-bs: 32 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "133" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp4-b16-c103-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp4-b16-c103-stp.yaml new file mode 100644 index 0000000000..89ca96b292 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp4-b16-c103-stp.yaml @@ -0,0 +1,145 @@ +schema: 2 +name: "gb200-fp4-glm5_8k1k_lowlat_2" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 72 + cuda-graph-max-bs: 72 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 9280 + mem-fraction-static: 0.8 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 6 + workers: 6 + gpus: 4 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 9600 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + load-balance-method: "round_robin" + moe-runner-backend: "flashinfer_trtllm" + enable-flashinfer-allreduce-fusion: true + mem-fraction-static: 0.94 + max-running-requests: 16 + cuda-graph-max-bs: 16 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "103" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp4-b22-c146-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp4-b22-c146-stp.yaml new file mode 100644 index 0000000000..19be6c44d8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp4-b22-c146-stp.yaml @@ -0,0 +1,145 @@ +schema: 2 +name: "gb200-fp4-glm5_8k1k_lowlat_1" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 72 + cuda-graph-max-bs: 72 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 9280 + mem-fraction-static: 0.8 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 6 + workers: 6 + gpus: 4 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 9600 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + load-balance-method: "round_robin" + moe-runner-backend: "flashinfer_trtllm" + enable-flashinfer-allreduce-fusion: true + mem-fraction-static: 0.94 + max-running-requests: 22 + cuda-graph-max-bs: 22 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "146" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p8d-p-tp4-ep4-dp4-d-tp4-b15-c130-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p8d-p-tp4-ep4-dp4-d-tp4-b15-c130-stp.yaml new file mode 100644 index 0000000000..c9a822d621 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p8d-p-tp4-ep4-dp4-d-tp4-b15-c130-stp.yaml @@ -0,0 +1,145 @@ +schema: 2 +name: "gb200-fp4-glm5_8k1k_lowlat_3" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 72 + cuda-graph-max-bs: 72 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 9280 + mem-fraction-static: 0.8 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 8 + workers: 8 + gpus: 4 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 9600 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + load-balance-method: "round_robin" + moe-runner-backend: "flashinfer_trtllm" + enable-flashinfer-allreduce-fusion: true + mem-fraction-static: 0.94 + max-running-requests: 15 + cuda-graph-max-bs: 15 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "130" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b512-c686-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b512-c686-stp.yaml new file mode 100644 index 0000000000..d3ecc2e714 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b512-c686-stp.yaml @@ -0,0 +1,150 @@ +schema: 2 +name: "gb200-fp4-glm5_8k1k_maxtpt_1" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 4 + workers: 4 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 72 + cuda-graph-max-bs: 72 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 9280 + mem-fraction-static: 0.812 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 9600 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 32 + data-parallel-size: 32 + expert-parallel-size: 32 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 0 + ep-dispatch-algorithm: "static" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + deepep-config: "/configs/deepep_config.json" + moe-runner-backend: "flashinfer_cutedsl" + enable-flashinfer-allreduce-fusion: false + mem-fraction-static: 0.812 + max-running-requests: 512 + cuda-graph-max-bs: 512 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "686" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b599-c989-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b599-c989-stp.yaml new file mode 100644 index 0000000000..bb8a45263b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b599-c989-stp.yaml @@ -0,0 +1,150 @@ +schema: 2 +name: "gb200-fp4-glm5_8k1k_maxtpt_0" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 4 + workers: 4 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 72 + cuda-graph-max-bs: 72 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 9280 + mem-fraction-static: 0.812 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 9600 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 32 + data-parallel-size: 32 + expert-parallel-size: 32 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 0 + ep-dispatch-algorithm: "static" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + deepep-config: "/configs/deepep_config.json" + moe-runner-backend: "flashinfer_cutedsl" + enable-flashinfer-allreduce-fusion: false + mem-fraction-static: 0.812 + max-running-requests: 599 + cuda-graph-max-bs: 599 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "989" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b914-c1497-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b914-c1497-stp.yaml new file mode 100644 index 0000000000..88df9a1435 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b914-c1497-stp.yaml @@ -0,0 +1,150 @@ +schema: 2 +name: "gb200-fp4-glm5_8k1k_maxtpt_2" + +# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). +# Upstream uses a single combined file with `zip_override_*` arrays expanded +# by srtctl across zip indices. We split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). +# All shared base envs and the prefill sglang_config are inlined here verbatim +# from the upstream `base:` block; the decode block is the upstream base plus +# the topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +dynamo: + source: + pypi: "1.2.1" + +slurm: + time_limit: "03:00:00" + +# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default +# that turns dynamo install + sglang weight load into a serial crawl; mem=0 +# grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 6 + workers: 6 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + max-running-requests: 72 + cuda-graph-max-bs: 72 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + context-length: 9280 + mem-fraction-static: 0.812 + nsa-prefill-backend: "trtllm" + nsa-decode-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + nodes: 8 + workers: 1 + gpus: 32 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + context-length: 9600 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + # Parallelism (override from upstream zip_override_*) + tensor-parallel-size: 32 + data-parallel-size: 32 + expert-parallel-size: 32 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 0 + ep-dispatch-algorithm: "static" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + deepep-config: "/configs/deepep_config.json" + moe-runner-backend: "flashinfer_cutedsl" + enable-flashinfer-allreduce-fusion: false + mem-fraction-static: 0.812 + max-running-requests: 914 + cuda-graph-max-bs: 914 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1497" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml new file mode 100644 index 0000000000..e8d70d7072 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml @@ -0,0 +1,242 @@ +schema: 2 +base: + name: gb200-fp4-glm5-mtp + model: + path: glm-5-fp4 + container: "lmsysorg/sglang:v0.5.13.post1-cu130" + precision: fp4 + identity: + model: + repo: nvidia/GLM-5-NVFP4 + revision: dc54ff55a7e9e71b85db953d8bc22eca894b44c6 + frameworks: + dynamo: 1.2.1 + sglang: 0.5.13.post1 + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: dynamo + dynamo: + source: + pypi: 1.2.1 + engine: sglang + roles: + prefill: + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + args: + served-model-name: GLM-5-FP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + max-running-requests: 136 + cuda-graph-max-bs: 136 + mem-fraction-static: 0.812 + context-length: 9280 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + args: + served-model-name: GLM-5-FP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + load-balance-method: round_robin + max-running-requests: 22 + cuda-graph-max-bs: 22 + chunked-prefill-size: 64 + context-length: 9280 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + mem-fraction-static: 0.918 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + health_check: + max_attempts: 360 + interval_seconds: 10 + benchmark: + type: sa-bench + req_rate: inf +# ################# 8k1k ################# +zip_override_mtp_8k1k_hightpt: + name: [8k1k-1p1d_dep16, 8k1k-2p1d_dep16, 8k1k-4p1d_dep16, 8k1k-8p1d_dep16] + roles: + prefill: + nodes: [1, 2, 4, 8] + workers: [1, 2, 4, 8] + decode: + nodes: 4 + workers: 1 + env: + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + args: + cuda-graph-max-bs: [91, 196, 451, 1354] + data-parallel-size: 16 + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + expert-parallel-size: 16 + load-balance-method: + max-running-requests: [91, 196, 451, 1354] + mem-fraction-static: 0.858 + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + moe-runner-backend: flashinfer_cutedsl + speculative-moe-a2a-backend: deepep + speculative-moe-runner-backend: deep_gemm + tensor-parallel-size: 16 + benchmark: + concurrencies: [[109], [398], [853], [2132]] + isl: 8192 + osl: 1024 + resources: {} +zip_override_mtp_8k1k_lowlat: + name: [8k1k-1p4d_tp4, 8k1k-1p4d_tp4-conc16, 8k1k-1p8d_tp4, 8k1k-1p8d_tp4-conc4, 8k1k-1p16d_tp4, 8k1k-1p16d_tp4-conc1] + roles: + prefill: + nodes: 1 + workers: 1 +# ################# 1k1k ################# + decode: + nodes: [4, 4, 8, 8, 16, 16] + workers: [4, 4, 8, 8, 16, 16] + args: + cuda-graph-max-bs: [22, 15, 9, 4, 4, 1] + max-running-requests: [22, 15, 9, 4, 4, 1] + benchmark: + concurrencies: [[104], [82], [87], [47], [74], [26]] + isl: 8192 + osl: 1024 + resources: {} +zip_override_mtp_1k1k_hightpt: + name: [1k1k-1p1d_dep16, 1k1k-1p1d_dep16-conc1024, 1k1k-1p1d_dep16-conc256, 1k1k-1p1d_dep32] + roles: + prefill: + nodes: 1 + workers: 1 + args: + context-length: 2112 + cuda-graph-max-bs: 512 + disable-cuda-graph: true + max-running-requests: 512 + mem-fraction-static: 0.83 + decode: + nodes: [4, 4, 4, 8] + workers: 1 + env: + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + args: + context-length: 2112 + cuda-graph-max-bs: [1627, 958, 239, 1122] + data-parallel-size: [16, 16, 16, 32] + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + expert-parallel-size: [16, 16, 16, 32] + max-running-requests: [1627, 958, 239, 1122] + mem-fraction-static: [0.858, 0.858, 0.858, 0.845] + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + moe-runner-backend: flashinfer_cutedsl + speculative-moe-a2a-backend: deepep + speculative-moe-runner-backend: deep_gemm + tensor-parallel-size: [16, 16, 16, 32] + benchmark: + concurrencies: [[1742], [1150], [302], [1263]] + isl: 1024 + osl: 1024 + resources: {} +zip_override_mtp_1k1k_lowlat: + name: [1k1k-1p16d_tp4-conc1, 1k1k-1p16d_tp4-conc16, 1k1k-1p16d_tp4-conc4] + roles: + prefill: + nodes: 1 + workers: 1 + args: + context-length: 2112 + cuda-graph-max-bs: 512 + disable-cuda-graph: true + max-running-requests: 512 + mem-fraction-static: 0.83 + decode: + nodes: 16 + workers: 16 + args: + context-length: 2112 + cuda-graph-max-bs: [1, 16, 4] + max-running-requests: [1, 16, 4] + benchmark: + concurrencies: [[26], [343], [94]] + isl: 1024 + osl: 1024 + resources: {} diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p1d-p-tp8-dp8-d-tp16-ep16-dp16-b1024-c1170-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p1d-p-tp8-dp8-d-tp16-ep16-dp16-b1024-c1170-stp.yaml new file mode 100644 index 0000000000..7bb0d44751 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p1d-p-tp8-dp8-d-tp16-ep16-dp16-b1024-c1170-stp.yaml @@ -0,0 +1,134 @@ +schema: 2 +name: gb200-fp8-glm5_1k1k_lowlat_1 + +# Combined upstream srt-slurm recipe split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention +# (see ../../../gb300-fp8/). All shared base envs and the prefill +# sglang_config are inlined here verbatim; the decode block is the shared +# base plus the topology-specific override for this concrete shape. + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.12" + precision: fp8 + +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + context-length: 2112 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + max-running-requests: 128 + cuda-graph-max-bs: 128 + mem-fraction-static: 0.7 + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + disable-radix-cache: true + + decode: + nodes: 4 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: deep_gemm + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + mem-fraction-static: 0.89 + context-length: 2112 + chunked-prefill-size: 64 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '1170' + random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p1d-p-tp8-dp8-d-tp16-ep16-dp16-b1815-c1955-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p1d-p-tp8-dp8-d-tp16-ep16-dp16-b1815-c1955-stp.yaml new file mode 100644 index 0000000000..3d33e0c2df --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p1d-p-tp8-dp8-d-tp16-ep16-dp16-b1815-c1955-stp.yaml @@ -0,0 +1,134 @@ +schema: 2 +name: gb200-fp8-glm5_1k1k_lowlat_0 + +# Combined upstream srt-slurm recipe split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention +# (see ../../../gb300-fp8/). All shared base envs and the prefill +# sglang_config are inlined here verbatim; the decode block is the shared +# base plus the topology-specific override for this concrete shape. + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.12" + precision: fp8 + +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + context-length: 2112 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + max-running-requests: 128 + cuda-graph-max-bs: 128 + mem-fraction-static: 0.7 + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + disable-radix-cache: true + + decode: + nodes: 4 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: deep_gemm + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + mem-fraction-static: 0.89 + context-length: 2112 + chunked-prefill-size: 64 + max-running-requests: 1815 + cuda-graph-max-bs: 1815 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '1955' + random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p1d-p-tp8-dp8-d-tp16-ep16-dp16-b256-c298-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p1d-p-tp8-dp8-d-tp16-ep16-dp16-b256-c298-stp.yaml new file mode 100644 index 0000000000..56f298cbf1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p1d-p-tp8-dp8-d-tp16-ep16-dp16-b256-c298-stp.yaml @@ -0,0 +1,134 @@ +schema: 2 +name: gb200-fp8-glm5_1k1k_lowlat_2 + +# Combined upstream srt-slurm recipe split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention +# (see ../../../gb300-fp8/). All shared base envs and the prefill +# sglang_config are inlined here verbatim; the decode block is the shared +# base plus the topology-specific override for this concrete shape. + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.12" + precision: fp8 + +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + context-length: 2112 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + max-running-requests: 128 + cuda-graph-max-bs: 128 + mem-fraction-static: 0.7 + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + disable-radix-cache: true + + decode: + nodes: 4 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: deep_gemm + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + mem-fraction-static: 0.89 + context-length: 2112 + chunked-prefill-size: 64 + max-running-requests: 256 + cuda-graph-max-bs: 256 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '298' + random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b1-c8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b1-c8-stp.yaml new file mode 100644 index 0000000000..670abfc016 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b1-c8-stp.yaml @@ -0,0 +1,125 @@ +schema: 2 +name: gb200-fp8-glm5_1k1k_lowlat_3 + +# Combined upstream srt-slurm recipe split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention +# (see ../../../gb300-fp8/). All shared base envs and the prefill +# sglang_config are inlined here verbatim; the decode block is the shared +# base plus the topology-specific override for this concrete shape. + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.12" + precision: fp8 + +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + context-length: 2112 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + max-running-requests: 128 + cuda-graph-max-bs: 128 + mem-fraction-static: 0.7 + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + disable-radix-cache: true + + decode: + nodes: 12 + workers: 6 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-flashinfer-allreduce-fusion: false + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + mem-fraction-static: 0.942 + context-length: 2112 + chunked-prefill-size: 64 + max-running-requests: 1 + cuda-graph-max-bs: 1 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '8' + random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b16-c72-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b16-c72-stp.yaml new file mode 100644 index 0000000000..11f1b500ca --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b16-c72-stp.yaml @@ -0,0 +1,125 @@ +schema: 2 +name: gb200-fp8-glm5_1k1k_lowlat_4 + +# Combined upstream srt-slurm recipe split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention +# (see ../../../gb300-fp8/). All shared base envs and the prefill +# sglang_config are inlined here verbatim; the decode block is the shared +# base plus the topology-specific override for this concrete shape. + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.12" + precision: fp8 + +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + context-length: 2112 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + max-running-requests: 128 + cuda-graph-max-bs: 128 + mem-fraction-static: 0.7 + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + disable-radix-cache: true + + decode: + nodes: 12 + workers: 6 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-flashinfer-allreduce-fusion: false + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + mem-fraction-static: 0.942 + context-length: 2112 + chunked-prefill-size: 64 + max-running-requests: 16 + cuda-graph-max-bs: 16 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '72' + random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b4-c20-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b4-c20-stp.yaml new file mode 100644 index 0000000000..16b51385f7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b4-c20-stp.yaml @@ -0,0 +1,125 @@ +schema: 2 +name: gb200-fp8-glm5_1k1k_lowlat_5 + +# Combined upstream srt-slurm recipe split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention +# (see ../../../gb300-fp8/). All shared base envs and the prefill +# sglang_config are inlined here verbatim; the decode block is the shared +# base plus the topology-specific override for this concrete shape. + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.12" + precision: fp8 + +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + context-length: 2112 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + max-running-requests: 128 + cuda-graph-max-bs: 128 + mem-fraction-static: 0.7 + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + disable-radix-cache: true + + decode: + nodes: 12 + workers: 6 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-flashinfer-allreduce-fusion: false + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + mem-fraction-static: 0.942 + context-length: 2112 + chunked-prefill-size: 64 + max-running-requests: 4 + cuda-graph-max-bs: 4 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '20' + random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-2p1d-p-tp8-dp8-d-tp32-ep32-dp32-b1024-c2161-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-2p1d-p-tp8-dp8-d-tp32-ep32-dp32-b1024-c2161-stp.yaml new file mode 100644 index 0000000000..8540b43211 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-2p1d-p-tp8-dp8-d-tp32-ep32-dp32-b1024-c2161-stp.yaml @@ -0,0 +1,136 @@ +schema: 2 +name: gb200-fp8-glm5_1k1k_hightpt_0 + +# Combined upstream srt-slurm recipe split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention +# (see ../../../gb300-fp8/). All shared base envs and the prefill +# sglang_config are inlined here verbatim; the decode block is the shared +# base plus the topology-specific override for this concrete shape. + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.12" + precision: fp8 + +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 4 + workers: 2 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + context-length: 2112 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + max-running-requests: 128 + cuda-graph-max-bs: 128 + mem-fraction-static: 0.7 + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + disable-radix-cache: true + + decode: + nodes: 8 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 32 + data-parallel-size: 32 + expert-parallel-size: 32 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: deep_gemm + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + mem-fraction-static: 0.886 + context-length: 2112 + chunked-prefill-size: 64 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '2161' + random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-1p8d-p-tp8-dp8-d-tp8-b1-c8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-1p8d-p-tp8-dp8-d-tp8-b1-c8-stp.yaml new file mode 100644 index 0000000000..5afee72140 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-1p8d-p-tp8-dp8-d-tp8-b1-c8-stp.yaml @@ -0,0 +1,123 @@ +schema: 2 +name: gb200-fp8-glm5_8k1k_lowlat_0 + +# Combined upstream srt-slurm recipe split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention +# (see ../../../gb300-fp8/). All shared base envs and the prefill +# sglang_config are inlined here verbatim; the decode block is the shared +# base plus the topology-specific override for this concrete shape. + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.12" + precision: fp8 + +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + context-length: 9280 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + max-running-requests: 128 + cuda-graph-max-bs: 128 + mem-fraction-static: 0.7 + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 16 + workers: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-flashinfer-allreduce-fusion: false + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + mem-fraction-static: 0.943 + context-length: 9280 + chunked-prefill-size: 64 + max-running-requests: 1 + cuda-graph-max-bs: 1 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '8' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-1p8d-p-tp8-dp8-d-tp8-b16-c90-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-1p8d-p-tp8-dp8-d-tp8-b16-c90-stp.yaml new file mode 100644 index 0000000000..d15781cf82 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-1p8d-p-tp8-dp8-d-tp8-b16-c90-stp.yaml @@ -0,0 +1,123 @@ +schema: 2 +name: gb200-fp8-glm5_8k1k_lowlat_1 + +# Combined upstream srt-slurm recipe split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention +# (see ../../../gb300-fp8/). All shared base envs and the prefill +# sglang_config are inlined here verbatim; the decode block is the shared +# base plus the topology-specific override for this concrete shape. + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.12" + precision: fp8 + +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + context-length: 9280 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + max-running-requests: 128 + cuda-graph-max-bs: 128 + mem-fraction-static: 0.7 + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 16 + workers: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-flashinfer-allreduce-fusion: false + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + mem-fraction-static: 0.943 + context-length: 9280 + chunked-prefill-size: 64 + max-running-requests: 16 + cuda-graph-max-bs: 16 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '90' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-1p8d-p-tp8-dp8-d-tp8-b4-c9-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-1p8d-p-tp8-dp8-d-tp8-b4-c9-stp.yaml new file mode 100644 index 0000000000..63ffa6d406 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-1p8d-p-tp8-dp8-d-tp8-b4-c9-stp.yaml @@ -0,0 +1,123 @@ +schema: 2 +name: gb200-fp8-glm5_8k1k_lowlat_2 + +# Combined upstream srt-slurm recipe split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention +# (see ../../../gb300-fp8/). All shared base envs and the prefill +# sglang_config are inlined here verbatim; the decode block is the shared +# base plus the topology-specific override for this concrete shape. + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.12" + precision: fp8 + +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + context-length: 9280 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + max-running-requests: 128 + cuda-graph-max-bs: 128 + mem-fraction-static: 0.7 + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 16 + workers: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-flashinfer-allreduce-fusion: false + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + mem-fraction-static: 0.943 + context-length: 9280 + chunked-prefill-size: 64 + max-running-requests: 4 + cuda-graph-max-bs: 4 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '9' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-3p1d-p-tp8-dp8-d-tp32-ep32-dp32-b256-c519-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-3p1d-p-tp8-dp8-d-tp32-ep32-dp32-b256-c519-stp.yaml new file mode 100644 index 0000000000..16ce66ec65 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-3p1d-p-tp8-dp8-d-tp32-ep32-dp32-b256-c519-stp.yaml @@ -0,0 +1,136 @@ +schema: 2 +name: gb200-fp8-glm5_8k1k_hightpt_0 + +# Combined upstream srt-slurm recipe split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention +# (see ../../../gb300-fp8/). All shared base envs and the prefill +# sglang_config are inlined here verbatim; the decode block is the shared +# base plus the topology-specific override for this concrete shape. + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.12" + precision: fp8 + +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 6 + workers: 3 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + context-length: 9280 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + max-running-requests: 128 + cuda-graph-max-bs: 128 + mem-fraction-static: 0.7 + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + disable-radix-cache: true + + decode: + nodes: 8 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 32 + data-parallel-size: 32 + expert-parallel-size: 32 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: deep_gemm + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + mem-fraction-static: 0.89 + context-length: 9280 + chunked-prefill-size: 64 + max-running-requests: 256 + cuda-graph-max-bs: 256 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '519' + random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-4p1d-p-tp8-dp8-d-tp16-ep16-dp16-b802-c1484-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-4p1d-p-tp8-dp8-d-tp16-ep16-dp16-b802-c1484-stp.yaml new file mode 100644 index 0000000000..3418b9674d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-4p1d-p-tp8-dp8-d-tp16-ep16-dp16-b802-c1484-stp.yaml @@ -0,0 +1,136 @@ +schema: 2 +name: gb200-fp8-glm5_8k1k_hightpt_1 + +# Combined upstream srt-slurm recipe split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention +# (see ../../../gb300-fp8/). All shared base envs and the prefill +# sglang_config are inlined here verbatim; the decode block is the shared +# base plus the topology-specific override for this concrete shape. + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.12" + precision: fp8 + +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 8 + workers: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + context-length: 9280 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + max-running-requests: 128 + cuda-graph-max-bs: 128 + mem-fraction-static: 0.7 + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + disable-radix-cache: true + + decode: + nodes: 4 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: deep_gemm + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + mem-fraction-static: 0.89 + context-length: 9280 + chunked-prefill-size: 64 + max-running-requests: 802 + cuda-graph-max-bs: 802 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '1484' + random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-5p1d-p-tp8-dp8-d-tp32-ep32-dp32-b828-c1688-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-5p1d-p-tp8-dp8-d-tp32-ep32-dp32-b828-c1688-stp.yaml new file mode 100644 index 0000000000..3ac21466f2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-5p1d-p-tp8-dp8-d-tp32-ep32-dp32-b828-c1688-stp.yaml @@ -0,0 +1,136 @@ +schema: 2 +name: gb200-fp8-glm5_8k1k_hightpt_2 + +# Combined upstream srt-slurm recipe split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention +# (see ../../../gb300-fp8/). All shared base envs and the prefill +# sglang_config are inlined here verbatim; the decode block is the shared +# base plus the topology-specific override for this concrete shape. + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.12" + precision: fp8 + +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 10 + workers: 5 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + context-length: 9280 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + max-running-requests: 128 + cuda-graph-max-bs: 128 + mem-fraction-static: 0.7 + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + disable-radix-cache: true + + decode: + nodes: 8 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 32 + data-parallel-size: 32 + expert-parallel-size: 32 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: deep_gemm + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + mem-fraction-static: 0.89 + context-length: 9280 + chunked-prefill-size: 64 + max-running-requests: 828 + cuda-graph-max-bs: 828 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '1688' + random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-7p1d-p-tp8-dp8-d-tp16-ep16-dp16-b1514-c2699-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-7p1d-p-tp8-dp8-d-tp16-ep16-dp16-b1514-c2699-stp.yaml new file mode 100644 index 0000000000..1a1159c45d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-7p1d-p-tp8-dp8-d-tp16-ep16-dp16-b1514-c2699-stp.yaml @@ -0,0 +1,136 @@ +schema: 2 +name: gb200-fp8-glm5_8k1k_hightpt_3 + +# Combined upstream srt-slurm recipe split into one flat yaml per concrete +# topology to match the InferenceX glm5 sglang convention +# (see ../../../gb300-fp8/). All shared base envs and the prefill +# sglang_config are inlined here verbatim; the decode block is the shared +# base plus the topology-specific override for this concrete shape. + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.12" + precision: fp8 + +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 14 + workers: 7 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + context-length: 9280 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + max-running-requests: 128 + cuda-graph-max-bs: 128 + mem-fraction-static: 0.7 + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + disable-radix-cache: true + + decode: + nodes: 4 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + + args: + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: deep_gemm + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + mem-fraction-static: 0.89 + context-length: 9280 + chunked-prefill-size: 64 + max-running-requests: 1514 + cuda-graph-max-bs: 1514 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '2699' + random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/fixed-seq-len/disagg-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/fixed-seq-len/disagg-mtp-variants.yaml new file mode 100644 index 0000000000..86b16f4940 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb200-fp8/fixed-seq-len/disagg-mtp-variants.yaml @@ -0,0 +1,164 @@ +schema: 2 +base: + name: gb200-fp8-glm5-mtp + model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.13.post1-cu130" + precision: fp8 + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: dynamo + dynamo: + source: + pypi: 1.2.1 + engine: sglang + roles: + prefill: + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + args: + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + enable-flashinfer-allreduce-fusion: false + max-running-requests: 136 + cuda-graph-max-bs: 136 + mem-fraction-static: 0.7 + context-length: 9280 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + args: + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + enable-flashinfer-allreduce-fusion: false + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + max-running-requests: 28 + cuda-graph-max-bs: 28 + chunked-prefill-size: 64 + mem-fraction-static: 0.94 + context-length: 9280 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + health_check: + max_attempts: 360 + interval_seconds: 10 + benchmark: + type: sa-bench + req_rate: inf + # MTP: EAGLE-style spec decoding is trained against chat-formatted + # inputs — keep the chat template on explicitly rather than relying + # on the srtctl schema default. + use_chat_template: true +# ################# 8k1k ################# +zip_override_mtp_8k1k_hightpt: + name: [8k1k-1p1d_dep16, 8k1k-2p1d_dep16, 8k1k-4p1d_dep16] + roles: + prefill: + nodes: [2, 4, 8] + workers: [1, 2, 4] + decode: + nodes: 4 + workers: 1 + env: + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + args: + cuda-graph-max-bs: [111, 234, 535] + data-parallel-size: 16 + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + expert-parallel-size: 16 + max-running-requests: [111, 234, 535] + mem-fraction-static: 0.85 + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + moe-runner-backend: deep_gemm + tensor-parallel-size: 16 + benchmark: + concurrencies: [[137], [620], [1305]] + isl: 8192 + osl: 1024 + resources: {} +zip_override_mtp_8k1k_lowlat: + name: [8k1k-1p4d_tp8, 8k1k-1p4d_tp8-conc56, 8k1k-1p8d_tp8, 8k1k-1p8d_tp8-conc12, 8k1k-1p8d_tp8-conc31] + roles: + prefill: + nodes: 2 + workers: 1 + decode: + nodes: [8, 8, 16, 16, 16] + workers: [4, 4, 8, 8, 8] + args: + cuda-graph-max-bs: [28, 8, 12, 1, 2] + max-running-requests: [28, 8, 12, 1, 2] + benchmark: + concurrencies: [[136], [56], [120], [12], [31]] + isl: 8192 + osl: 1024 + resources: {} diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/1k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b1-c32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/1k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b1-c32-mtp.yaml new file mode 100644 index 0000000000..b474ef9af4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/1k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b1-c32-mtp.yaml @@ -0,0 +1,148 @@ +schema: 2 +name: gb300-fp8-glm5-mtp_1k1k_lowlat_1 + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +frontend: + type: dynamo +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 17 + workers: 17 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + moe-runner-backend: flashinfer_trtllm + max-running-requests: 1 + cuda-graph-max-bs: 1 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '32' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/1k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b32-c512x256x128x64-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/1k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b32-c512x256x128x64-mtp.yaml new file mode 100644 index 0000000000..3e67380443 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/1k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b32-c512x256x128x64-mtp.yaml @@ -0,0 +1,148 @@ +schema: 2 +name: gb300-fp8-glm5-mtp_1k1k_lowlat_0 + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +frontend: + type: dynamo +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 17 + workers: 17 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + moe-runner-backend: flashinfer_trtllm + max-running-requests: 32 + cuda-graph-max-bs: 32 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: 512x256x128x64 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/1k1k/disagg-4p1d-p-tp4-dp4-d-tp56-ep56-dp56-b5600-c5700-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/1k1k/disagg-4p1d-p-tp4-dp4-d-tp56-ep56-dp56-b5600-c5700-mtp.yaml new file mode 100644 index 0000000000..9c9586deeb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/1k1k/disagg-4p1d-p-tp4-dp4-d-tp56-ep56-dp56-b5600-c5700-mtp.yaml @@ -0,0 +1,156 @@ +schema: 2 +name: gb300-fp8-glm5-mtp_1k1k_hightpt_4 + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 4 + workers: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 14 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 56 + expert-parallel-size: 56 + data-parallel-size: 56 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 24 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 5600 + cuda-graph-max-bs: 100 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '5700' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/1k1k/disagg-6p1d-p-tp4-dp4-d-tp48-ep48-dp48-b6144-c6500-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/1k1k/disagg-6p1d-p-tp4-dp4-d-tp48-ep48-dp48-b6144-c6500-mtp.yaml new file mode 100644 index 0000000000..4ff5626b5c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/1k1k/disagg-6p1d-p-tp4-dp4-d-tp48-ep48-dp48-b6144-c6500-mtp.yaml @@ -0,0 +1,156 @@ +schema: 2 +name: gb300-fp8-glm5-mtp_1k1k_hightpt_3 + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 6 + workers: 6 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 12 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 48 + expert-parallel-size: 48 + data-parallel-size: 48 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 6144 + cuda-graph-max-bs: 128 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '6500' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-10p1d-p-tp4-dp4-d-tp32-ep32-dp32-b1280-c1300-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-10p1d-p-tp4-dp4-d-tp32-ep32-dp32-b1280-c1300-mtp.yaml new file mode 100644 index 0000000000..5c3d6531bc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-10p1d-p-tp4-dp4-d-tp32-ep32-dp32-b1280-c1300-mtp.yaml @@ -0,0 +1,156 @@ +schema: 2 +name: gb300-fp8-glm5-mtp_8k1k_hightpt_2 + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 10 + workers: 10 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 8 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 32 + expert-parallel-size: 32 + data-parallel-size: 32 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 1280 + cuda-graph-max-bs: 40 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '1300' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-12p1d-p-tp4-dp4-d-tp24-ep24-dp24-b1680-c1700-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-12p1d-p-tp4-dp4-d-tp24-ep24-dp24-b1680-c1700-mtp.yaml new file mode 100644 index 0000000000..14ec800d1b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-12p1d-p-tp4-dp4-d-tp24-ep24-dp24-b1680-c1700-mtp.yaml @@ -0,0 +1,156 @@ +schema: 2 +name: gb300-fp8-glm5-mtp_8k1k_hightpt_1 + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 12 + workers: 12 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 6 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 24 + expert-parallel-size: 24 + data-parallel-size: 24 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 1680 + cuda-graph-max-bs: 70 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '1700' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-14p1d-p-tp4-dp4-d-tp16-ep16-dp16-b2800-c2800-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-14p1d-p-tp4-dp4-d-tp16-ep16-dp16-b2800-c2800-mtp.yaml new file mode 100644 index 0000000000..12c7bc66d9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-14p1d-p-tp4-dp4-d-tp16-ep16-dp16-b2800-c2800-mtp.yaml @@ -0,0 +1,156 @@ +schema: 2 +name: gb300-fp8-glm5-mtp_8k1k_hightpt_0 + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 14 + workers: 14 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 4 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 16 + expert-parallel-size: 16 + data-parallel-size: 16 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 2800 + cuda-graph-max-bs: 175 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '2800' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b1-c24-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b1-c24-mtp.yaml new file mode 100644 index 0000000000..f3d05fdece --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b1-c24-mtp.yaml @@ -0,0 +1,148 @@ +schema: 2 +name: gb300-fp8-glm5-mtp_8k1k_lowlat_2 + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +frontend: + type: dynamo +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 17 + workers: 17 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + moe-runner-backend: flashinfer_trtllm + max-running-requests: 1 + cuda-graph-max-bs: 1 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '24' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b8-c128x64x32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b8-c128x64x32-mtp.yaml new file mode 100644 index 0000000000..7dffe5d05d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b8-c128x64x32-mtp.yaml @@ -0,0 +1,148 @@ +schema: 2 +name: gb300-fp8-glm5-mtp_8k1k_lowlat_1 + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +frontend: + type: dynamo +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 17 + workers: 17 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + moe-runner-backend: flashinfer_trtllm + max-running-requests: 8 + cuda-graph-max-bs: 8 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: 128x64x32 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-1p9d-p-tp4-dp4-d-tp4-b15-c150-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-1p9d-p-tp4-dp4-d-tp4-b15-c150-mtp.yaml new file mode 100644 index 0000000000..82a64c86e0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-1p9d-p-tp4-dp4-d-tp4-b15-c150-mtp.yaml @@ -0,0 +1,148 @@ +schema: 2 +name: gb300-fp8-glm5-mtp_8k1k_lowlat_0 + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +frontend: + type: dynamo +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 9 + workers: 9 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + moe-runner-backend: flashinfer_trtllm + max-running-requests: 15 + cuda-graph-max-bs: 15 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '150' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-8p1d-p-tp4-dp4-d-tp40-ep40-dp40-b880-c900-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-8p1d-p-tp4-dp4-d-tp40-ep40-dp40-b880-c900-mtp.yaml new file mode 100644 index 0000000000..326437f1da --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-8p1d-p-tp4-dp4-d-tp40-ep40-dp40-b880-c900-mtp.yaml @@ -0,0 +1,156 @@ +schema: 2 +name: gb300-fp8-glm5-mtp_8k1k_hightpt_3 + +model: + path: glm-5.1-fp8 + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: fp8 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 9 +dynamo: + source: + pypi: 1.2.1 + +engine: sglang +roles: + prefill: + nodes: 8 + workers: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + + # Other flags + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 10 + workers: 1 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). + # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + + args: + # Model configuration + served-model-name: GLM-5.1-FP8 + trust-remote-code: true + + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + + # Disaggregation mode + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + # moe-runner-backend: "cutedsl" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + tensor-parallel-size: 40 + expert-parallel-size: 40 + data-parallel-size: 40 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + ep-num-redundant-experts: 24 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 880 + cuda-graph-max-bs: 22 + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '900' diff --git a/benchmarks/multi_node/srt-slurm-recipes/tilert/glm5.1/b200-fp8/agentic/disagg-1p1d-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/tilert/b200-fp8/agentx/disagg-1p1d-tp8-mtp.yaml similarity index 79% rename from benchmarks/multi_node/srt-slurm-recipes/tilert/glm5.1/b200-fp8/agentic/disagg-1p1d-tp8-mtp.yaml rename to benchmarks/multi_node/srt-slurm-recipes/glm5.1/tilert/b200-fp8/agentx/disagg-1p1d-tp8-mtp.yaml index 1e78863b8c..cfed0bde13 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/tilert/glm5.1/b200-fp8/agentic/disagg-1p1d-tp8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.1/tilert/b200-fp8/agentx/disagg-1p1d-tp8-mtp.yaml @@ -1,3 +1,4 @@ +schema: 2 name: "b200-glm51-tilert-agentic-disagg-1p1d-tp8-mtp" model: @@ -26,13 +27,6 @@ slurm: resources: gpu_type: "b200" gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - frontend: type: tilert-router enable_multiple_frontends: false @@ -42,7 +36,7 @@ frontend: parser: none queue-timeout: 1800 -backend: +engine: type: tilert prefill_container: "tilert-prefill" model_profile: "glm5" @@ -54,21 +48,30 @@ backend: transport: nixl with_mtp: true speculative_tokens: 1 - prefill_environment: &common_environment - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache/hub" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" - PYTHONUNBUFFERED: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - decode_environment: - <<: *common_environment - tilert_config: - prefill: +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: &common_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + args: enforce-eager: true trust-remote-code: true gpu-memory-utilization: 0.75 + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + <<: *common_environment srun_options: container-writable: "" container-remap-root: "" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml new file mode 100644 index 0000000000..26c7d02b07 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml @@ -0,0 +1,127 @@ +# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 1. +# +# One TP8 worker serves prefill and decode on a single node. The sweep matrix +# supplies the concurrency list; agentic_srt.sh replays every point against +# this one server. Acceptance is pinned to the golden thinking-on AL for three +# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). +schema: 2 +name: agg-b200-tp8-c1-mtp +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-20260910-00840301 + frameworks: + dynamo: 1.5.0.dev20260909 + sglang: 0.0.0.dev1+g008403017 +resources: + gpu_type: b200 + gpus_per_node: 8 +dynamo: + install: false +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_SIMULATE_ACC_LEN: '2.99' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 110 + hicache-io-backend: direct + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +health_check: + max_attempts: 1440 + interval_seconds: 10 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 +frontend: + type: dynamo + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml new file mode 100644 index 0000000000..1425efb008 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml @@ -0,0 +1,127 @@ +# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 4. +# +# One TP8 worker serves prefill and decode on a single node. The sweep matrix +# supplies the concurrency list; agentic_srt.sh replays every point against +# this one server. Acceptance is pinned to the golden thinking-on AL for three +# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). +schema: 2 +name: agg-b200-tp8-c4-mtp +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-20260910-00840301 + frameworks: + dynamo: 1.5.0.dev20260909 + sglang: 0.0.0.dev1+g008403017 +resources: + gpu_type: b200 + gpus_per_node: 8 +dynamo: + install: false +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_SIMULATE_ACC_LEN: '2.99' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 110 + hicache-io-backend: direct + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +health_check: + max_attempts: 1440 + interval_seconds: 10 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 +frontend: + type: dynamo + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml new file mode 100644 index 0000000000..a88d2ca313 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml @@ -0,0 +1,127 @@ +# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 8. +# +# One TP8 worker serves prefill and decode on a single node. The sweep matrix +# supplies the concurrency list; agentic_srt.sh replays every point against +# this one server. Acceptance is pinned to the golden thinking-on AL for three +# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). +schema: 2 +name: agg-b200-tp8-c8-mtp +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-20260910-00840301 + frameworks: + dynamo: 1.5.0.dev20260909 + sglang: 0.0.0.dev1+g008403017 +resources: + gpu_type: b200 + gpus_per_node: 8 +dynamo: + install: false +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_SIMULATE_ACC_LEN: '2.99' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 110 + hicache-io-backend: direct + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +health_check: + max_attempts: 1440 + interval_seconds: 10 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 +frontend: + type: dynamo + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml new file mode 100644 index 0000000000..ba9897dfdf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml @@ -0,0 +1,202 @@ +schema: 2 +name: disagg-b200-1p1d-dep8-dep8-c64-mtp +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-20260910-00840301 + frameworks: + dynamo: 1.5.0.dev20260909 + sglang: 0.0.0.dev1+g008403017 +resources: + gpu_type: b200 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +dynamo: + install: false +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 110 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 128 + cuda-graph-max-bs-decode: 128 + chunked-prefill-size: 64 + context-length: 1048576 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutedsl + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.85 + disaggregation-decode-extra-slots: 0 + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + speculative-moe-a2a-backend: deepep + speculative-moe-runner-backend: deep_gemm +health_check: + max_attempts: 1440 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml new file mode 100644 index 0000000000..b45fad092c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml @@ -0,0 +1,194 @@ +schema: 2 +name: disagg-b200-1p4d-dep8-tp4-c48-mtp +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-20260910-00840301 + frameworks: + dynamo: 1.5.0.dev20260909 + sglang: 0.0.0.dev1+g008403017 +resources: + gpu_type: b200 + gpus_per_node: 8 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +dynamo: + install: false +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '3600' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 110 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + decode: + nodes: 2 + workers: 4 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '3600' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 16 + cuda-graph-max-bs-decode: 16 + chunked-prefill-size: 64 + context-length: 1048576 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.9 + disaggregation-decode-extra-slots: 0 + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true +health_check: + max_attempts: 1440 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml new file mode 100644 index 0000000000..e3badbfd06 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: agg-gb200-tp8-c2-mtp +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + frameworks: + dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + sglang: nightly-dev-cu13-20260805-211ee642 +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +dynamo: + install: true + source: + rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd +engine: sglang +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_SIMULATE_ACC_LEN: '3.33' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 10 + cuda-graph-max-bs: 10 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 4 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 5 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 135 + hicache-io-backend: direct + enable-metrics: true +health_check: + max_attempts: 1440 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml new file mode 100644 index 0000000000..fc70de159f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: agg-gb200-tp8-c4-mtp +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + frameworks: + dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + sglang: nightly-dev-cu13-20260805-211ee642 +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +dynamo: + install: true + source: + rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd +engine: sglang +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_SIMULATE_ACC_LEN: '3.33' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 10 + cuda-graph-max-bs: 10 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 4 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 5 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 135 + hicache-io-backend: direct + enable-metrics: true +health_check: + max_attempts: 1440 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml new file mode 100644 index 0000000000..1ccd98f838 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: agg-gb200-tp8-c8-mtp +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + frameworks: + dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + sglang: nightly-dev-cu13-20260805-211ee642 +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +dynamo: + install: true + source: + rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd +engine: sglang +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_SIMULATE_ACC_LEN: '3.33' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 10 + cuda-graph-max-bs: 10 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 4 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 5 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 135 + hicache-io-backend: direct + enable-metrics: true +health_check: + max_attempts: 1440 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml new file mode 100644 index 0000000000..220d8e3f63 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml @@ -0,0 +1,149 @@ +schema: 2 +name: gb200-fp4-glm5.2-agentx-agg +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:v0.5.17-cu130 + frameworks: + dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + sglang: 0.5.17 +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 +dynamo: + install: true + source: + rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd +engine: sglang +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_SIMULATE_ACC_LEN: '2.99' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self + NVSHMEM_REMOTE_TRANSPORT: none + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 10 + cuda-graph-max-bs: 10 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + bf16-gemm-backend: cutedsl + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.83 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 135 + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true +health_check: + max_attempts: 1440 + interval_seconds: 10 +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 +benchmark: + type: custom + concurrencies: [1] + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml new file mode 100644 index 0000000000..5ce18e3183 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml @@ -0,0 +1,203 @@ +schema: 2 +name: disagg-gb200-1p4d-dep8-tp4-c48-mtp +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + frameworks: + dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + sglang: nightly-dev-cu13-20260805-211ee642 +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +dynamo: + install: true + source: + rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + max-running-requests: 16 + cuda-graph-max-bs: 16 + disable-cuda-graph: true + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 135 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 2 + enable-metrics: true + enable-cache-report: true + decode: + nodes: 4 + workers: 4 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 16 + cuda-graph-max-bs: 16 + chunked-prefill-size: 64 + context-length: 1048576 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.9 + disaggregation-decode-extra-slots: 0 + enable-metrics: true + enable-cache-report: true +health_check: + max_attempts: 1440 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml new file mode 100644 index 0000000000..49543a48f5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml @@ -0,0 +1,203 @@ +schema: 2 +name: disagg-gb200-1p6d-dep8-tp4-c45-mtp +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + frameworks: + dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + sglang: nightly-dev-cu13-20260805-211ee642 +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +dynamo: + install: true + source: + rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + max-running-requests: 16 + cuda-graph-max-bs: 16 + disable-cuda-graph: true + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 135 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 2 + enable-metrics: true + enable-cache-report: true + decode: + nodes: 6 + workers: 6 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 16 + cuda-graph-max-bs: 16 + chunked-prefill-size: 64 + context-length: 1048576 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.9 + disaggregation-decode-extra-slots: 0 + enable-metrics: true + enable-cache-report: true +health_check: + max_attempts: 1440 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml new file mode 100644 index 0000000000..96d9c6fff5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml @@ -0,0 +1,211 @@ +schema: 2 +name: disagg-gb200-2p1d-dep8-dep16-c128-mtp +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + frameworks: + dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + sglang: nightly-dev-cu13-20260805-211ee642 +resources: + gpu_type: gb200 + gpus_per_node: 4 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +dynamo: + install: true + source: + rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd +engine: sglang +roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + max-running-requests: 16 + cuda-graph-max-bs: 16 + disable-cuda-graph: true + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 100 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 2 + enable-metrics: true + enable-cache-report: true + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: 144 + cuda-graph-max-bs: 144 + chunked-prefill-size: 64 + context-length: 1048576 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutedsl + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.85 + disaggregation-decode-extra-slots: 0 + enable-metrics: true + enable-cache-report: true + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + speculative-moe-a2a-backend: deepep + speculative-moe-runner-backend: deep_gemm +health_check: + max_attempts: 1440 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml new file mode 100644 index 0000000000..081d41f4f3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml @@ -0,0 +1,232 @@ +schema: 2 +base: + name: gb200-fp4-glm5.2-agentx + model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 + identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa + container: + image: lmsysorg/sglang:v0.5.17-cu130 + frameworks: + dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + sglang: 0.5.17 + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-kv-events: true + router-temperature: 0 + kv-cache-block-size: 64 + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 + dynamo: + install: true + source: + rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + engine: sglang + roles: + prefill: + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '2048' + SGLANG_SIMULATE_ACC_LEN: '2.99' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 4 + attn-cp-size: 1 + enable-prefill-cp: false + enable-dsa-cache-layer-split: false + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + load-balance-method: total_tokens + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + max-total-tokens: 1048576 + max-running-requests: 64 + cuda-graph-max-bs: 64 + disable-cuda-graph: true + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + bf16-gemm-backend: cutedsl + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.83 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 135 + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + kv_events: true + decode: + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '2048' + SGLANG_SIMULATE_ACC_LEN: '2.99' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 64 + cuda-graph-max-bs: 64 + chunked-prefill-size: 64 + context-length: 1048576 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + bf16-gemm-backend: cutedsl + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.9 + disaggregation-decode-extra-slots: 0 + watchdog-timeout: 1800 + enable-metrics: true + enable-cache-report: true + kv_events: true + health_check: + max_attempts: 1440 + interval_seconds: 10 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + +zip_override_mtp_agentx_frontier: + name: [agentx-1p1d-tp4-hicache] + resources: {} + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + decode: + nodes: 1 + workers: 1 + gpus: 4 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg.yaml new file mode 100644 index 0000000000..4969a2689e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/agg.yaml @@ -0,0 +1,128 @@ +# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on GB300 +# (single aggregated worker, TP8, EAGLE MTP + hierarchical-cache KV). +# +# Flat single-variant schema the agentic CI flow expects: applied via +# CONFIG_FILE= with no zip_override selector, so it must NOT be +# base:-wrapped. Concurrency is not a recipe field; the GHA matrix fans out +# one job per concurrency from the master-config conc-list into agentic_srt.sh. +schema: 2 +name: gb300-fp4-glm5.2-agentx-agg +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 +frontend: + type: dynamo + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 +dynamo: + install: true + source: + rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd +engine: sglang +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_SIMULATE_ACC_LEN: '3.33' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self + NVSHMEM_REMOTE_TRANSPORT: none + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 10 + cuda-graph-max-bs: 10 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 4 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 5 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 135 + hicache-io-backend: direct + enable-metrics: true + enable-cache-report: true +health_check: + max_attempts: 1440 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-mtp-variants.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-mtp-variants.yaml new file mode 100644 index 0000000000..93585243bd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-mtp-variants.yaml @@ -0,0 +1,234 @@ +schema: 2 +base: + name: gb300-fp4-glm5.2-agentx + model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 + resources: + gpu_type: gb300 + gpus_per_node: 4 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 + dynamo: + install: true + source: + rev: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + engine: sglang + roles: + prefill: + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + max-running-requests: 16 + cuda-graph-max-bs: 16 + disable-cuda-graph: true + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 135 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 2 + enable-metrics: true + enable-cache-report: true + decode: + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 16 + cuda-graph-max-bs: 16 + chunked-prefill-size: 64 + context-length: 1048576 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.9 + disaggregation-decode-extra-slots: 0 + enable-metrics: true + enable-cache-report: true + health_check: + max_attempts: 1440 + interval_seconds: 10 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 64 + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' +# ################# agentx ################# +zip_override_mtp_agentx_hightpt: + name: [agentx-2p1d_dep16-c128, agentx-2p1d_dep16-c192] + roles: + prefill: + nodes: 4 + workers: 2 + gpus: 8 + args: + data-parallel-size: 8 + expert-parallel-size: 8 + max-prefill-tokens: 8192 + tensor-parallel-size: 8 + decode: + nodes: 4 + workers: 1 + gpus: 16 + args: + cuda-graph-max-bs: 144 + data-parallel-size: 16 + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + expert-parallel-size: 16 + max-running-requests: 144 + mem-fraction-static: 0.85 + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + moe-runner-backend: flashinfer_cutedsl + speculative-moe-a2a-backend: deepep + speculative-moe-runner-backend: deep_gemm + tensor-parallel-size: 16 + resources: {} +zip_override_mtp_agentx_lowlat: + name: [agentx-1p2d_tp4-c48, agentx-1p4d_tp4-c48, agentx-1p6d_tp4-c45] + resources: {} + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + decode: + nodes: [2, 4, 6] + workers: [2, 4, 6] + gpus: 4 diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml new file mode 100644 index 0000000000..85193d57af --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml @@ -0,0 +1,176 @@ +schema: 2 +name: "disagg-h200-1p1d-pcp8-tp8-dp8-mtp6-hicache" + +# 1P1D GLM-5.2 FP8 AgentX topology on H200 with EAGLE MTP and prefill-side +# hierarchical KV cache. The decode-side KV cache remains GPU-resident. + +model: + path: "hf:zai-org/GLM-5.2-FP8" + container: "lmsysorg/sglang:v0.5.16-cu130" + precision: "fp8" + +dynamo: + install: true + source: + wheel: "1.3.0.dev1" + +resources: + gpu_type: h200 + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + env: + DYN_ROUTER_TEMPERATURE: "10000000" + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + PIP_BREAK_SYSTEM_PACKAGES: "1" + PYTHONUNBUFFERED: "1" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_USE_TOPK_V2: "1" + SGLANG_TIMEOUT_KEEP_ALIVE: "900" + args: + host: 0.0.0.0 + model-path: /model/ + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tool-call-parser: glm47 + reasoning-parser: glm45 + tp-size: 8 + ep-size: 1 + attn-cp-size: 8 + enable-prefill-cp: true + cp-strategy: interleave + enable-dsa-cache-layer-split: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + kv-cache-dtype: fp8_e4m3 + context-length: 1048576 + max-total-tokens: 1048576 + chunked-prefill-size: 32768 + mem-fraction-static: 0.85 + max-running-requests: 32 + page-size: 64 + enable-hierarchical-cache: true + hicache-size: 64 + hicache-mem-layout: layer_first + hicache-io-backend: kernel + hicache-write-policy: write_back + speculative-algorithm: EAGLE + speculative-num-steps: 6 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + watchdog-timeout: 86400 + stream-interval: 60 + enable-metrics: true + enable-cache-report: true + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + PIP_BREAK_SYSTEM_PACKAGES: "1" + PYTHONUNBUFFERED: "1" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_USE_TOPK_V2: "1" + SGLANG_TIMEOUT_KEEP_ALIVE: "900" + SGLANG_SIMULATE_ACC_LEN: "3.78" + SGLANG_SIMULATE_ACC_METHOD: "match-expected" + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + + args: + host: 0.0.0.0 + model-path: /model/ + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tool-call-parser: glm47 + reasoning-parser: glm45 + tp-size: 8 + dp-size: 8 + ep-size: 1 + enable-dp-attention: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + kv-cache-dtype: fp8_e4m3 + dsa-decode-backend: flashmla_kv + context-length: 1048576 + max-total-tokens: 1048576 + mem-fraction-static: 0.85 + max-running-requests: 200 + page-size: 64 + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 6 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + watchdog-timeout: 86400 + stream-interval: 60 + enable-metrics: true + enable-cache-report: true + +sbatch_directives: + mem: "0" + +srun_options: + mem: "0" + container-remap-root: "" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml new file mode 100644 index 0000000000..e3b8169397 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml @@ -0,0 +1,176 @@ +schema: 2 +name: "disagg-h200-2p2d-pcp8-tp8-dp8-mtp" + +# Refresh of the 2P2D topology from Actions run 30133535261 with EAGLE MTP. +# The released SGLang image does not combine MTP with HiSparse, so decode KV +# cache remains GPU-resident. MTP uses the committed GLM-5.2 golden thinking-on +# acceptance length for three speculative steps. + +model: + path: "hf:zai-org/GLM-5.2-FP8" + container: "lmsysorg/sglang:v0.5.16-cu130" + precision: "fp8" + +dynamo: + install: true + source: + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +resources: + gpu_type: h200 + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + env: + DYN_ROUTER_TEMPERATURE: "10000000" + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 8 + env: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + PIP_BREAK_SYSTEM_PACKAGES: "1" + PYTHONUNBUFFERED: "1" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_USE_TOPK_V2: "1" + SGLANG_TIMEOUT_KEEP_ALIVE: "900" + args: + host: 0.0.0.0 + model-path: /model/ + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tool-call-parser: glm47 + reasoning-parser: glm45 + tp-size: 8 + ep-size: 1 + attn-cp-size: 8 + enable-prefill-cp: true + cp-strategy: interleave + enable-dsa-cache-layer-split: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + kv-cache-dtype: fp8_e4m3 + context-length: 1048576 + max-total-tokens: 1048576 + chunked-prefill-size: 32768 + mem-fraction-static: 0.85 + max-running-requests: 32 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + watchdog-timeout: 86400 + stream-interval: 60 + enable-metrics: true + enable-cache-report: true + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + PIP_BREAK_SYSTEM_PACKAGES: "1" + PYTHONUNBUFFERED: "1" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_USE_TOPK_V2: "1" + SGLANG_TIMEOUT_KEEP_ALIVE: "900" + SGLANG_SIMULATE_ACC_LEN: "2.99" + SGLANG_SIMULATE_ACC_METHOD: "match-expected" + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + + args: + host: 0.0.0.0 + model-path: /model/ + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tool-call-parser: glm47 + reasoning-parser: glm45 + tp-size: 8 + dp-size: 8 + ep-size: 1 + enable-dp-attention: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + kv-cache-dtype: fp8_e4m3 + dsa-decode-backend: flashmla_kv + context-length: 1048576 + max-total-tokens: 1048576 + mem-fraction-static: 0.85 + max-running-requests: 200 + page-size: 64 + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + watchdog-timeout: 86400 + stream-interval: 60 + enable-metrics: true + enable-cache-report: true + +sbatch_directives: + mem: "0" + +srun_options: + mem: "0" + container-remap-root: "" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + +benchmark: + type: custom + # Replaced with CONC_LIST by the launcher before submission. + concurrencies: [1] + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml new file mode 100644 index 0000000000..30ac636a36 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml @@ -0,0 +1,189 @@ +schema: 2 +name: dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + precision: fp4 + +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + request_plane: tcp + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_FUSED_DSA_METADATA: "1" + TRTLLM_DSA_INDEXER_BF16: "1" + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + args: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + tensor_parallel_size: 4 + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: *server_environment + args: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 5] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 5 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 8 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + placement: + node: head +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml new file mode 100644 index 0000000000..0d47084914 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml @@ -0,0 +1,186 @@ +schema: 2 +name: dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + request_plane: tcp + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_FUSED_DSA_METADATA: "1" + TRTLLM_DSA_INDEXER_BF16: "1" + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + args: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: EQUAL_PROGRESS + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + tensor_parallel_size: 4 + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: *server_environment + args: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 1 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 8 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + placement: + node: head +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml new file mode 100644 index 0000000000..1b9210f789 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml @@ -0,0 +1,189 @@ +schema: 2 +name: dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + precision: fp4 + +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + request_plane: tcp + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_FUSED_DSA_METADATA: "1" + TRTLLM_DSA_INDEXER_BF16: "1" + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + args: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + tensor_parallel_size: 4 + decode: + nodes: 4 + workers: 4 + gpus: 4 + + env: *server_environment + args: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 2 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + placement: + node: head +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml new file mode 100644 index 0000000000..bd5670b6a8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml @@ -0,0 +1,189 @@ +schema: 2 +name: dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + precision: fp4 + +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + request_plane: tcp + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false + +roles: + prefill: + nodes: 3 + workers: 3 + gpus: 4 + env: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_FUSED_DSA_METADATA: "1" + TRTLLM_DSA_INDEXER_BF16: "1" + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + args: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + tensor_parallel_size: 4 + decode: + nodes: 4 + workers: 4 + gpus: 4 + + env: *server_environment + args: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 5] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 5 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + placement: + node: head +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml new file mode 100644 index 0000000000..3d8c7f4b5a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml @@ -0,0 +1,190 @@ +schema: 2 +name: dynamo-disagg-gb300-4p1d-dep8-c227-b16-mtp3 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + precision: fp4 + +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + request_plane: tcp + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false + +roles: + prefill: + nodes: 4 + workers: 4 + gpus: 4 + env: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_FUSED_DSA_METADATA: "1" + TRTLLM_DSA_INDEXER_BF16: "1" + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "1.99" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + args: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: *server_environment + args: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 8, 16] + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.9 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 16 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 8 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + placement: + node: head +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml new file mode 100644 index 0000000000..ff5b3fe29c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml @@ -0,0 +1,190 @@ +schema: 2 +name: dynamo-disagg-gb300-5p1d-dep16-c260-b16-mtp3 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + precision: fp4 + +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + request_plane: tcp + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false + +roles: + prefill: + nodes: 5 + workers: 5 + gpus: 4 + env: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_FUSED_DSA_METADATA: "1" + TRTLLM_DSA_INDEXER_BF16: "1" + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "1.99" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + args: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: *server_environment + args: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 8, 16] + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.9 + tokens_per_block: 64 + max_batch_size: 16 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + enable_iter_perf_stats: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 16 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + placement: + node: head +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + placement: + node: dedicated diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p15d-p-tp4-dp4-d-tp4-b16-c16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p15d-p-tp4-dp4-d-tp4-b16-c16-stp.yaml new file mode 100644 index 0000000000..6b782ab2b8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p15d-p-tp4-dp4-d-tp4-b16-c16-stp.yaml @@ -0,0 +1,174 @@ +schema: 2 +name: "gb300-fp4-glm5_8k1k_lowlat_3" + +# Ported from upstream srt-slurm recipes/gb300-fp4/glm5.yaml (PR #152). +# Upstream uses a single combined file with `zip_override_*` arrays +# expanded by srtctl across zip indices. We split into one flat yaml +# per concrete topology to match the InferenceX dsv4 sglang convention +# (see ../deepseek-v4/8k1k/*.yaml). All shared base envs and the +# prefill sglang_config are inlined here verbatim from the upstream +# `base:` block; the decode block is the upstream base plus the +# topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +# Released dynamo wheel; unlike hash-based sources, this recipe does not +# require a persistent /configs/dynamo-wheels build cache. +dynamo: + source: + pypi: "1.1.0" + +slurm: + time_limit: "03:00:00" + +# Mirror dsv4 sglang recipes: cpus-per-task=144 avoids the 1-CPU +# default that turns dynamo install + sglang weight load into a serial +# crawl; mem=0 grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + # Model configuration + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + + # Other flags + # disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 15 + workers: 15 + gpus: 4 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + # Model configuration + served-model-name: "GLM-5-FP4" + trust-remote-code: true + + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + # disable-shared-experts-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + # Parallelism (override from upstream zip_override_*_lowlat) + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + + moe-runner-backend: "flashinfer_trtllm" + max-running-requests: 16 + cuda-graph-max-bs: 16 + + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "16" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b1-c12-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b1-c12-stp.yaml new file mode 100644 index 0000000000..b0cb87ce49 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b1-c12-stp.yaml @@ -0,0 +1,174 @@ +schema: 2 +name: "gb300-fp4-glm5_8k1k_lowlat_4" + +# Ported from upstream srt-slurm recipes/gb300-fp4/glm5.yaml (PR #152). +# Upstream uses a single combined file with `zip_override_*` arrays +# expanded by srtctl across zip indices. We split into one flat yaml +# per concrete topology to match the InferenceX dsv4 sglang convention +# (see ../deepseek-v4/8k1k/*.yaml). All shared base envs and the +# prefill sglang_config are inlined here verbatim from the upstream +# `base:` block; the decode block is the upstream base plus the +# topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +# Released dynamo wheel; unlike hash-based sources, this recipe does not +# require a persistent /configs/dynamo-wheels build cache. +dynamo: + source: + pypi: "1.1.0" + +slurm: + time_limit: "03:00:00" + +# Mirror dsv4 sglang recipes: cpus-per-task=144 avoids the 1-CPU +# default that turns dynamo install + sglang weight load into a serial +# crawl; mem=0 grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + # Model configuration + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + + # Other flags + # disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 17 + workers: 17 + gpus: 4 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + # Model configuration + served-model-name: "GLM-5-FP4" + trust-remote-code: true + + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + # disable-shared-experts-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + # Parallelism (override from upstream zip_override_*_lowlat) + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + + moe-runner-backend: "flashinfer_trtllm" + max-running-requests: 1 + cuda-graph-max-bs: 1 + + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "12" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p3d-p-tp4-dp4-d-tp4-b128-c128-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p3d-p-tp4-dp4-d-tp4-b128-c128-stp.yaml new file mode 100644 index 0000000000..43c1ed393a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p3d-p-tp4-dp4-d-tp4-b128-c128-stp.yaml @@ -0,0 +1,174 @@ +schema: 2 +name: "gb300-fp4-glm5_8k1k_lowlat_0" + +# Ported from upstream srt-slurm recipes/gb300-fp4/glm5.yaml (PR #152). +# Upstream uses a single combined file with `zip_override_*` arrays +# expanded by srtctl across zip indices. We split into one flat yaml +# per concrete topology to match the InferenceX dsv4 sglang convention +# (see ../deepseek-v4/8k1k/*.yaml). All shared base envs and the +# prefill sglang_config are inlined here verbatim from the upstream +# `base:` block; the decode block is the upstream base plus the +# topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +# Released dynamo wheel; unlike hash-based sources, this recipe does not +# require a persistent /configs/dynamo-wheels build cache. +dynamo: + source: + pypi: "1.1.0" + +slurm: + time_limit: "03:00:00" + +# Mirror dsv4 sglang recipes: cpus-per-task=144 avoids the 1-CPU +# default that turns dynamo install + sglang weight load into a serial +# crawl; mem=0 grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + # Model configuration + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + + # Other flags + # disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 3 + workers: 3 + gpus: 4 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + # Model configuration + served-model-name: "GLM-5-FP4" + trust-remote-code: true + + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + # disable-shared-experts-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + # Parallelism (override from upstream zip_override_*_lowlat) + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + + moe-runner-backend: "flashinfer_trtllm" + max-running-requests: 128 + cuda-graph-max-bs: 128 + + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "128" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p5d-p-tp4-dp4-d-tp4-b64-c64-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p5d-p-tp4-dp4-d-tp4-b64-c64-stp.yaml new file mode 100644 index 0000000000..c874fc2d32 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p5d-p-tp4-dp4-d-tp4-b64-c64-stp.yaml @@ -0,0 +1,174 @@ +schema: 2 +name: "gb300-fp4-glm5_8k1k_lowlat_1" + +# Ported from upstream srt-slurm recipes/gb300-fp4/glm5.yaml (PR #152). +# Upstream uses a single combined file with `zip_override_*` arrays +# expanded by srtctl across zip indices. We split into one flat yaml +# per concrete topology to match the InferenceX dsv4 sglang convention +# (see ../deepseek-v4/8k1k/*.yaml). All shared base envs and the +# prefill sglang_config are inlined here verbatim from the upstream +# `base:` block; the decode block is the upstream base plus the +# topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +# Released dynamo wheel; unlike hash-based sources, this recipe does not +# require a persistent /configs/dynamo-wheels build cache. +dynamo: + source: + pypi: "1.1.0" + +slurm: + time_limit: "03:00:00" + +# Mirror dsv4 sglang recipes: cpus-per-task=144 avoids the 1-CPU +# default that turns dynamo install + sglang weight load into a serial +# crawl; mem=0 grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + # Model configuration + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + + # Other flags + # disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 5 + workers: 5 + gpus: 4 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + # Model configuration + served-model-name: "GLM-5-FP4" + trust-remote-code: true + + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + # disable-shared-experts-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + # Parallelism (override from upstream zip_override_*_lowlat) + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + + moe-runner-backend: "flashinfer_trtllm" + max-running-requests: 64 + cuda-graph-max-bs: 64 + + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "64" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p9d-p-tp4-dp4-d-tp4-b32-c32-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p9d-p-tp4-dp4-d-tp4-b32-c32-stp.yaml new file mode 100644 index 0000000000..313271b555 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p9d-p-tp4-dp4-d-tp4-b32-c32-stp.yaml @@ -0,0 +1,174 @@ +schema: 2 +name: "gb300-fp4-glm5_8k1k_lowlat_2" + +# Ported from upstream srt-slurm recipes/gb300-fp4/glm5.yaml (PR #152). +# Upstream uses a single combined file with `zip_override_*` arrays +# expanded by srtctl across zip indices. We split into one flat yaml +# per concrete topology to match the InferenceX dsv4 sglang convention +# (see ../deepseek-v4/8k1k/*.yaml). All shared base envs and the +# prefill sglang_config are inlined here verbatim from the upstream +# `base:` block; the decode block is the upstream base plus the +# topology-specific override from this zip index. + +model: + path: "glm-5-fp4" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp4" + +# Released dynamo wheel; unlike hash-based sources, this recipe does not +# require a persistent /configs/dynamo-wheels build cache. +dynamo: + source: + pypi: "1.1.0" + +slurm: + time_limit: "03:00:00" + +# Mirror dsv4 sglang recipes: cpus-per-task=144 avoids the 1-CPU +# default that turns dynamo install + sglang weight load into a serial +# crawl; mem=0 grants whole-node memory. +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + # Model configuration + served-model-name: "GLM-5-FP4" + trust-remote-code: true + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + + # Other flags + # disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + nodes: 9 + workers: 9 + gpus: 4 + + env: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + MC_TE_METRIC: "true" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_MOE_NVFP4_DISPATCH: "1" + + args: + # Model configuration + served-model-name: "GLM-5-FP4" + trust-remote-code: true + + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + fp4-gemm-backend: "flashinfer_cutlass" + + # Detokenizer + skip-tokenizer-init: true + stream-interval: 30 + + # Other flags + # disable-shared-experts-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + # Parallelism (override from upstream zip_override_*_lowlat) + tensor-parallel-size: 4 + expert-parallel-size: 1 + data-parallel-size: 1 + enable-flashinfer-allreduce-fusion: true + + moe-runner-backend: "flashinfer_trtllm" + max-running-requests: 32 + cuda-graph-max-bs: 32 + + + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "32" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep16-b32-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep16-b32-eplb0-mtp3.yaml new file mode 100644 index 0000000000..0ffff53e6d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep16-b32-eplb0-mtp3.yaml @@ -0,0 +1,119 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep16_batch32_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '615' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep16-b64-eplb0-mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep16-b64-eplb0-mtp1.yaml new file mode 100644 index 0000000000..21d02a5aa8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep16-b64-eplb0-mtp1.yaml @@ -0,0 +1,123 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep16_batch64_eplb0_mtp1 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 128 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '1229' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b16-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b16-eplb0-mtp0.yaml new file mode 100644 index 0000000000..6a1228d9ef --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b16-eplb0-mtp0.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep32_batch16_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 8 + workers: 1 + gpus: 32 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '666' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b16-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b16-eplb0-mtp3.yaml new file mode 100644 index 0000000000..223e5eb3d7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b16-eplb0-mtp3.yaml @@ -0,0 +1,117 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep32_batch16_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 8 + workers: 1 + gpus: 32 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 64 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '666' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b32-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b32-eplb0-mtp0.yaml new file mode 100644 index 0000000000..9126667ee8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b32-eplb0-mtp0.yaml @@ -0,0 +1,113 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep32_batch32_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 8 + workers: 1 + gpus: 32 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '1229' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b4-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b4-eplb0-mtp3.yaml new file mode 100644 index 0000000000..bc8d784c47 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b4-eplb0-mtp3.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep32_batch4_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 8 + workers: 1 + gpus: 32 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '180' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b8-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b8-eplb0-mtp3.yaml new file mode 100644 index 0000000000..a6735556c5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b8-eplb0-mtp3.yaml @@ -0,0 +1,116 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep32_batch8_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 8 + workers: 1 + gpus: 32 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '308' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep8-b512-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep8-b512-eplb0-mtp0.yaml new file mode 100644 index 0000000000..5582a7f14f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep8-b512-eplb0-mtp0.yaml @@ -0,0 +1,173 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep8_batch512_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 2 + workers: 1 + gpus: 8 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 512 + max_num_tokens: 512 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + - 264 + - 272 + - 280 + - 288 + - 296 + - 304 + - 312 + - 320 + - 328 + - 336 + - 344 + - 352 + - 360 + - 368 + - 376 + - 384 + - 392 + - 400 + - 408 + - 416 + - 424 + - 432 + - 440 + - 448 + - 456 + - 464 + - 472 + - 480 + - 488 + - 496 + - 504 + - 512 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '4301' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b1-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b1-eplb0-mtp0.yaml new file mode 100644 index 0000000000..47863f828c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b1-eplb0-mtp0.yaml @@ -0,0 +1,110 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch1_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 8 + workers: 4 + gpus: 8 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + allreduce_strategy: MNNVL +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '4' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b1-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b1-eplb0-mtp3.yaml new file mode 100644 index 0000000000..f27c877c64 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b1-eplb0-mtp3.yaml @@ -0,0 +1,116 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch1_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 8 + workers: 4 + gpus: 8 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + allreduce_strategy: MNNVL +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '8' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b16-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b16-eplb0-mtp0.yaml new file mode 100644 index 0000000000..1ff0f181b8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b16-eplb0-mtp0.yaml @@ -0,0 +1,112 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch16_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 8 + workers: 4 + gpus: 8 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + allreduce_strategy: MNNVL +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '84' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b16-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b16-eplb0-mtp3.yaml new file mode 100644 index 0000000000..7507abc5bc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b16-eplb0-mtp3.yaml @@ -0,0 +1,118 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch16_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 8 + workers: 4 + gpus: 8 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 64 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + allreduce_strategy: MNNVL +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '92' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b32-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b32-eplb0-mtp0.yaml new file mode 100644 index 0000000000..c426e9eef7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b32-eplb0-mtp0.yaml @@ -0,0 +1,114 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch32_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 8 + workers: 4 + gpus: 8 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + allreduce_strategy: MNNVL +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '168' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b4-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b4-eplb0-mtp0.yaml new file mode 100644 index 0000000000..2957f3e02f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b4-eplb0-mtp0.yaml @@ -0,0 +1,110 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch4_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 8 + workers: 4 + gpus: 8 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 4 + max_num_tokens: 4 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + allreduce_strategy: MNNVL +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '20' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b4-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b4-eplb0-mtp3.yaml new file mode 100644 index 0000000000..7b926eb80e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b4-eplb0-mtp3.yaml @@ -0,0 +1,116 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch4_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 8 + workers: 4 + gpus: 8 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + allreduce_strategy: MNNVL +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '24' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b64-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b64-eplb0-mtp0.yaml new file mode 100644 index 0000000000..05ce6a902b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b64-eplb0-mtp0.yaml @@ -0,0 +1,118 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch64_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 8 + workers: 4 + gpus: 8 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 64 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + allreduce_strategy: MNNVL +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '284' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b8-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b8-eplb0-mtp3.yaml new file mode 100644 index 0000000000..fec045f4ac --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b8-eplb0-mtp3.yaml @@ -0,0 +1,117 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch8_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 8 + workers: 4 + gpus: 8 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + allreduce_strategy: MNNVL +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '40' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp0.yaml new file mode 100644 index 0000000000..b7a3cb1c9a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp0.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen5tep4_batch1_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 5 + workers: 5 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '5' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp3.yaml new file mode 100644 index 0000000000..8517c3a4fa --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp3.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen5tep4_batch1_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 5 + workers: 5 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '10' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp0.yaml new file mode 100644 index 0000000000..d148ed4f95 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp0.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen5tep4_batch4_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 5 + workers: 5 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 4 + max_num_tokens: 4 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '25' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep16-b128-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep16-b128-eplb0-mtp0.yaml new file mode 100644 index 0000000000..99c6554463 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep16-b128-eplb0-mtp0.yaml @@ -0,0 +1,125 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx2dep4_gen1dep16_batch128_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 128 + max_num_tokens: 128 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '2151' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep16-b128-eplb0-mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep16-b128-eplb0-mtp1.yaml new file mode 100644 index 0000000000..130fa6bb6b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep16-b128-eplb0-mtp1.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx2dep4_gen1dep16_batch128_eplb0_mtp1 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 128 + max_num_tokens: 256 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '2253' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep16-b256-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep16-b256-eplb0-mtp0.yaml new file mode 100644 index 0000000000..ced81fc285 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep16-b256-eplb0-mtp0.yaml @@ -0,0 +1,141 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx2dep4_gen1dep16_batch256_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 256 + max_num_tokens: 256 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '4301' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep16-b256-eplb256-mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep16-b256-eplb256-mtp1.yaml new file mode 100644 index 0000000000..6f27353da4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep16-b256-eplb256-mtp1.yaml @@ -0,0 +1,150 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx2dep4_gen1dep16_batch256_eplb256_mtp1 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 256 + max_num_tokens: 512 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + moe_config: + backend: CUTEDSL + load_balancer: + layer_updates_per_iter: 1 + num_slots: 256 + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '4301' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep32-b64-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep32-b64-eplb0-mtp0.yaml new file mode 100644 index 0000000000..0bd5a72238 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep32-b64-eplb0-mtp0.yaml @@ -0,0 +1,117 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx2dep4_gen1dep32_batch64_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 8 + workers: 1 + gpus: 32 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 64 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '2151' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep8-b512-eplb0-mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep8-b512-eplb0-mtp1.yaml new file mode 100644 index 0000000000..414148bd60 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep8-b512-eplb0-mtp1.yaml @@ -0,0 +1,179 @@ +schema: 2 +name: glm5_nvfp4_ISL1K_OSL1K_ctx2dep4_gen1dep8_batch512_eplb0_mtp1 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 1064 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + decode: + nodes: 2 + workers: 1 + gpus: 8 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 8 + moe_expert_parallel_size: 8 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 512 + max_num_tokens: 1024 + max_seq_len: 2088 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + - 264 + - 272 + - 280 + - 288 + - 296 + - 304 + - 312 + - 320 + - 328 + - 336 + - 344 + - 352 + - 360 + - 368 + - 376 + - 384 + - 392 + - 400 + - 408 + - 416 + - 424 + - 432 + - 440 + - 448 + - 456 + - 464 + - 472 + - 480 + - 488 + - 496 + - 504 + - 512 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + concurrencies: '4301' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-12p1d-dep4-dep16-b128-eplb0-mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-12p1d-dep4-dep16-b128-eplb0-mtp1.yaml new file mode 100644 index 0000000000..fe66f7fa3e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-12p1d-dep4-dep16-b128-eplb0-mtp1.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx12dep4_gen1dep16_batch128_eplb0_mtp1 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 12 + workers: 12 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 128 + max_num_tokens: 256 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '2253' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p1d-dep4-dep32-b2-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p1d-dep4-dep32-b2-eplb0-mtp3.yaml new file mode 100644 index 0000000000..fdba0398a7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p1d-dep4-dep32-b2-eplb0-mtp3.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen1dep32_batch2_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 8 + workers: 1 + gpus: 32 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 8 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '90' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp0.yaml new file mode 100644 index 0000000000..654eb57ff7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp0.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch1_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 5 + workers: 5 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '5' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp3.yaml new file mode 100644 index 0000000000..9d4fcf0159 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp3.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch1_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 5 + workers: 5 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '5' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b16-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b16-eplb0-mtp0.yaml new file mode 100644 index 0000000000..d3a6d07593 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b16-eplb0-mtp0.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch16_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 5 + workers: 5 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '105' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b2-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b2-eplb0-mtp0.yaml new file mode 100644 index 0000000000..be34a46f87 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b2-eplb0-mtp0.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch2_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 5 + workers: 5 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 2 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '10' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b2-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b2-eplb0-mtp3.yaml new file mode 100644 index 0000000000..933fc48b88 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b2-eplb0-mtp3.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch2_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 5 + workers: 5 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 8 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '15' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp0.yaml new file mode 100644 index 0000000000..9a159c271b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp0.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch4_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 5 + workers: 5 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 4 + max_num_tokens: 4 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '25' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp3.yaml new file mode 100644 index 0000000000..9752b64434 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp3.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch4_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 5 + workers: 5 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '30' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b8-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b8-eplb0-mtp0.yaml new file mode 100644 index 0000000000..4ac2290c52 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b8-eplb0-mtp0.yaml @@ -0,0 +1,110 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch8_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 5 + workers: 5 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 8 + max_num_tokens: 8 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '50' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep4-dep32-b4-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep4-dep32-b4-eplb0-mtp3.yaml new file mode 100644 index 0000000000..bff3a586d4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep4-dep32-b4-eplb0-mtp3.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx2dep4_gen1dep32_batch4_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 8 + workers: 1 + gpus: 32 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 4 + max_num_tokens: 16 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '180' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep4-dep32-b8-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep4-dep32-b8-eplb0-mtp0.yaml new file mode 100644 index 0000000000..7229c6e76d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep4-dep32-b8-eplb0-mtp0.yaml @@ -0,0 +1,110 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx2dep4_gen1dep32_batch8_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 8 + workers: 1 + gpus: 32 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 8 + max_num_tokens: 8 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '308' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-4p1d-dep4-dep32-b16-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-4p1d-dep4-dep32-b16-eplb0-mtp0.yaml new file mode 100644 index 0000000000..e9072e706f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-4p1d-dep4-dep32-b16-eplb0-mtp0.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx4dep4_gen1dep32_batch16_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 4 + workers: 4 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 8 + workers: 1 + gpus: 32 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '615' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-4p1d-dep4-dep32-b8-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-4p1d-dep4-dep32-b8-eplb0-mtp3.yaml new file mode 100644 index 0000000000..6aa843481d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-4p1d-dep4-dep32-b8-eplb0-mtp3.yaml @@ -0,0 +1,116 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx4dep4_gen1dep32_batch8_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 4 + workers: 4 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 8 + workers: 1 + gpus: 32 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '333' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: false +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-5p1d-dep4-dep16-b64-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-5p1d-dep4-dep16-b64-eplb0-mtp0.yaml new file mode 100644 index 0000000000..adc683120b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-5p1d-dep4-dep16-b64-eplb0-mtp0.yaml @@ -0,0 +1,117 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx5dep4_gen1dep16_batch64_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 5 + workers: 5 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 64 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '1127' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-6p1d-dep4-dep16-b32-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-6p1d-dep4-dep16-b32-eplb0-mtp3.yaml new file mode 100644 index 0000000000..c8a8ed856d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-6p1d-dep4-dep16-b32-eplb0-mtp3.yaml @@ -0,0 +1,119 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx6dep4_gen1dep16_batch32_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 6 + workers: 6 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '615' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-6p1d-dep4-dep32-b32-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-6p1d-dep4-dep32-b32-eplb0-mtp0.yaml new file mode 100644 index 0000000000..85e113e9d3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-6p1d-dep4-dep32-b32-eplb0-mtp0.yaml @@ -0,0 +1,113 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx6dep4_gen1dep32_batch32_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 6 + workers: 6 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 8 + workers: 1 + gpus: 32 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '1229' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep4-dep32-b16-eplb0-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep4-dep32-b16-eplb0-mtp3.yaml new file mode 100644 index 0000000000..a0e0810f5b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep4-dep32-b16-eplb0-mtp3.yaml @@ -0,0 +1,117 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx7dep4_gen1dep32_batch16_eplb0_mtp3 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 7 + workers: 7 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + decode: + nodes: 8 + workers: 1 + gpus: 32 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 16 + max_num_tokens: 64 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '666' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep4-dep16-b64-eplb0-mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep4-dep16-b64-eplb0-mtp1.yaml new file mode 100644 index 0000000000..96119745ba --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep4-dep16-b64-eplb0-mtp1.yaml @@ -0,0 +1,123 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx8dep4_gen1dep16_batch64_eplb0_mtp1 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 8 + workers: 8 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + trust_remote_code: true + max_batch_size: 64 + max_num_tokens: 128 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 1 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '1127' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-9p1d-dep4-dep16-b128-eplb0-mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-9p1d-dep4-dep16-b128-eplb0-mtp0.yaml new file mode 100644 index 0000000000..b1332efdff --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-9p1d-dep4-dep16-b128-eplb0-mtp0.yaml @@ -0,0 +1,125 @@ +schema: 2 +name: glm5_nvfp4_ISL8K_OSL1K_ctx9dep4_gen1dep16_batch128_eplb0_mtp0 +model: + path: nvidia/GLM-5-NVFP4 + container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + precision: fp4 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 9 + workers: 9 + gpus: 4 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + enable_attention_dp: true + disable_overlap_scheduler: true + trust_remote_code: true + max_batch_size: 2 + max_num_tokens: 16640 + max_seq_len: 8232 + print_iter_log: true + cuda_graph_config: + moe_config: + backend: CUTEDSL + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + ENROOT_ALLOW_DEV: 'yes' + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + TLLM_LOG_LEVEL: INFO + TRTLLM_ENABLE_PDL: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' + args: + tensor_parallel_size: 16 + moe_expert_parallel_size: 16 + pipeline_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_batch_size: 128 + max_num_tokens: 128 + max_seq_len: 9256 + print_iter_log: true + stream_interval: 100 + num_postprocess_workers: 4 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + cache_transceiver_config: + backend: UCX + max_tokens_in_buffer: 16384 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '2151' + req_rate: 'inf' + num_prompts_mult: 16 + use_chat_template: true +frontend: + type: dynamo + enable_multiple_frontends: true +health_check: + max_attempts: 360 + interval_seconds: 10 +dynamo: + install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p1d-dep4-dep8-c308-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p1d-dep4-dep8-c308-stp.yaml new file mode 100644 index 0000000000..7b70090991 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p1d-dep4-dep8-c308-stp.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: disagg-B200-1p1d-dep4-dep8-c308-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + source: + rev: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.92 + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9256 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 308 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p4d-dep4-tep8-c24-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p4d-dep4-tep8-c24-stp.yaml new file mode 100644 index 0000000000..9efed38c1d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p4d-dep4-tep8-c24-stp.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: disagg-B200-1p4d-dep4-tep8-c24-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + source: + rev: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + nodes: 4 + workers: 4 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 4 + max_num_tokens: 4 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 24 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p4d-dep4-tep8-c4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p4d-dep4-tep8-c4-stp.yaml new file mode 100644 index 0000000000..33e18fd915 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p4d-dep4-tep8-c4-stp.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: disagg-B200-1p4d-dep4-tep8-c4-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + source: + rev: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + nodes: 4 + workers: 4 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 4 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p5d-dep4-tep4-c115-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p5d-dep4-tep4-c115-stp.yaml new file mode 100644 index 0000000000..b03bbea144 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p5d-dep4-tep4-c115-stp.yaml @@ -0,0 +1,129 @@ +schema: 2 +name: disagg-B200-1p5d-dep4-tep4-c115-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + source: + rev: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + nodes: 3 + workers: 5 + gpus: 4 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 115 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p5d-dep4-tep4-c195-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p5d-dep4-tep4-c195-stp.yaml new file mode 100644 index 0000000000..82ddd6648c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p5d-dep4-tep4-c195-stp.yaml @@ -0,0 +1,131 @@ +schema: 2 +name: disagg-B200-1p5d-dep4-tep4-c195-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + source: + rev: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + nodes: 3 + workers: 5 + gpus: 4 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 195 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p5d-dep4-tep4-c30-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p5d-dep4-tep4-c30-stp.yaml new file mode 100644 index 0000000000..939cefcefd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p5d-dep4-tep4-c30-stp.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: disagg-B200-1p5d-dep4-tep4-c30-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + source: + rev: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + nodes: 3 + workers: 5 + gpus: 4 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 4 + max_num_tokens: 4 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 30 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p5d-dep4-tep4-c5-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p5d-dep4-tep4-c5-stp.yaml new file mode 100644 index 0000000000..8638021360 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p5d-dep4-tep4-c5-stp.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: disagg-B200-1p5d-dep4-tep4-c5-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + source: + rev: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + nodes: 3 + workers: 5 + gpus: 4 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 5 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p5d-dep4-tep4-c60-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p5d-dep4-tep4-c60-stp.yaml new file mode 100644 index 0000000000..c5cc9ceab4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-1p5d-dep4-tep4-c60-stp.yaml @@ -0,0 +1,128 @@ +schema: 2 +name: disagg-B200-1p5d-dep4-tep4-c60-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + source: + rev: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + nodes: 3 + workers: 5 + gpus: 4 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 8 + max_num_tokens: 8 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 60 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-2p1d-dep4-dep8-c615-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-2p1d-dep4-dep8-c615-stp.yaml new file mode 100644 index 0000000000..028a790246 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-2p1d-dep4-dep8-c615-stp.yaml @@ -0,0 +1,135 @@ +schema: 2 +name: disagg-B200-2p1d-dep4-dep8-c615-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + source: + rev: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 4 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.92 + max_batch_size: 64 + max_num_tokens: 64 + max_seq_len: 9256 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 615 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-3p1d-dep4-dep8-c1127-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-3p1d-dep4-dep8-c1127-stp.yaml new file mode 100644 index 0000000000..503f6c83a8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-3p1d-dep4-dep8-c1127-stp.yaml @@ -0,0 +1,143 @@ +schema: 2 +name: disagg-B200-3p1d-dep4-dep8-c1127-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + source: + rev: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 3 + gpus: 4 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.92 + max_batch_size: 128 + max_num_tokens: 128 + max_seq_len: 9256 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 1127 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-4p1d-dep4-dep8-c2151-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-4p1d-dep4-dep8-c2151-stp.yaml new file mode 100644 index 0000000000..d7a6aad22c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/trtllm/b200-fp4/8k1k/disagg-4p1d-dep4-dep8-c2151-stp.yaml @@ -0,0 +1,159 @@ +schema: 2 +name: disagg-B200-4p1d-dep4-dep8-c2151-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + source: + rev: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 +engine: trtllm +roles: + prefill: + nodes: 2 + workers: 4 + gpus: 4 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.92 + max_batch_size: 256 + max_num_tokens: 256 + max_seq_len: 9256 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 2151 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..6522cc6a16 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep16-stp.yaml @@ -0,0 +1,104 @@ +schema: 2 +name: "vllm-disagg-gb200-1p1d-dep4-dep16" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.2.1 +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 1024 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.9 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 256 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + all2all-backend: "flashinfer_nvlink_one_sided" + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 128 + max-cudagraph-capture-size: 384 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "4096x6144" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep8-stp.yaml new file mode 100644 index 0000000000..1a59f8b23f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep8-stp.yaml @@ -0,0 +1,104 @@ +schema: 2 +name: "vllm-disagg-gb200-1p1d-dep4-dep8" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.2.1 +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 1024 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.93 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 1024 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + all2all-backend: "flashinfer_nvlink_one_sided" + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 128 + max-cudagraph-capture-size: 1024 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "4096x12288" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tp8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tp8-stp.yaml new file mode 100644 index 0000000000..a4214a70c8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tp8-stp.yaml @@ -0,0 +1,100 @@ +schema: 2 +name: "kimi-vllm-disagg-gb200-1p4d-dep4-tp8" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.2.1 +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 1024 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.93 + + decode: + nodes: 8 + workers: 4 + gpus: 8 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + enable-expert-parallel: false + max-model-len: 3072 + max-num-seqs: 1024 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "4x8x32x128" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-1p4d-dep4-tep4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-1p4d-dep4-tep4-stp.yaml new file mode 100644 index 0000000000..18e1f3383b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-1p4d-dep4-tep4-stp.yaml @@ -0,0 +1,100 @@ +schema: 2 +name: "kimi-vllm-disagg-gb200-1p4d-dep4-tep4" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.2.1 +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 64 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.93 + + decode: + nodes: 4 + workers: 4 + gpus: 4 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 16 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 16 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "128" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-1p4d-dep4-tp8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-1p4d-dep4-tp8-stp.yaml new file mode 100644 index 0000000000..9b3d5455bd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-1p4d-dep4-tp8-stp.yaml @@ -0,0 +1,100 @@ +schema: 2 +name: "kimi-vllm-disagg-gb200-1p4d-dep4-tp8" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.2.1 +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 64 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.93 + + decode: + nodes: 8 + workers: 4 + gpus: 8 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + enable-expert-parallel: false + max-model-len: 10240 + max-num-seqs: 16 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 16 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4x8x16x32x256" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-3p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-3p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..0bb8ce54c9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-3p1d-dep4-dep16-stp.yaml @@ -0,0 +1,103 @@ +schema: 2 +name: "kimi-vllm-disagg-gb200-3p1d-dep4-dep16" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.2.1 +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 3 + workers: 3 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 64 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.93 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 256 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + all2all-backend: "flashinfer_nvlink_one_sided" + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 256 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1024" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-6p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-6p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..2851f85805 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-6p1d-dep4-dep16-stp.yaml @@ -0,0 +1,103 @@ +schema: 2 +name: "kimi-vllm-disagg-gb200-6p1d-dep4-dep16" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.2.1 +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 6 + workers: 6 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 64 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.93 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 512 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + all2all-backend: "flashinfer_nvlink_one_sided" + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 512 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "3072" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-8p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-8p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..b2c516630c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-8p1d-dep4-dep16-stp.yaml @@ -0,0 +1,104 @@ +schema: 2 +name: "kimi-vllm-disagg-gb200-8p1d-dep4-dep16" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.2.1 +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 8 + workers: 8 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 64 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.93 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 256 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + all2all-backend: "flashinfer_nvlink_one_sided" + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 512 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "6144" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-1p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-1p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..dd068cf8b6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-1p1d-dep4-dep16-stp.yaml @@ -0,0 +1,103 @@ +schema: 2 +name: "kimi-vllm-disagg-gb300-1p1d-dep4-dep16" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.3.0.dev20260601 +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 1024 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.9 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 192 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + all2all-backend: "flashinfer_nvlink_one_sided" + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 256 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "2048x4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-1p1d-dep4-dep24-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-1p1d-dep4-dep24-stp.yaml new file mode 100644 index 0000000000..16de544580 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-1p1d-dep4-dep24-stp.yaml @@ -0,0 +1,103 @@ +schema: 2 +name: "kimi-vllm-disagg-gb300-1p1d-dep4-dep24" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.3.0.dev20260601 +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 1024 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.9 + + decode: + nodes: 6 + workers: 1 + gpus: 24 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 24 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 86 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + all2all-backend: "flashinfer_nvlink_one_sided" + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 128 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1024" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-1p2d-dep4-dep4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-1p2d-dep4-dep4-stp.yaml new file mode 100644 index 0000000000..99c500f2ae --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-1p2d-dep4-dep4-stp.yaml @@ -0,0 +1,103 @@ +schema: 2 +name: "kimi-vllm-disagg-gb300-1p2d-dep4-dep4" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.3.0.dev20260601 +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 1024 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.9 + + decode: + nodes: 2 + workers: 2 + gpus: 4 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 768 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + all2all-backend: "flashinfer_nvlink_one_sided" + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "6144" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-1p7d-tep4-tp4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-1p7d-tep4-tp4-stp.yaml new file mode 100644 index 0000000000..a02cfed747 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-1p7d-tep4-tp4-stp.yaml @@ -0,0 +1,98 @@ +schema: 2 +name: "kimi-vllm-disagg-gb300-1p7d-tep4-tp4" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.3.0.dev20260601 +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 1024 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.9 + + decode: + nodes: 7 + workers: 7 + gpus: 4 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 3072 + max-num-seqs: 1024 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + all2all-backend: "flashinfer_nvlink_one_sided" + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "8x16x32x64x128" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-2p3d-dep4-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-2p3d-dep4-dep8-stp.yaml new file mode 100644 index 0000000000..0845b72635 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-2p3d-dep4-dep8-stp.yaml @@ -0,0 +1,103 @@ +schema: 2 +name: "kimi-vllm-disagg-gb300-2p3d-dep4-dep8" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.3.0.dev20260601 +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 1024 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.9 + + decode: + nodes: 6 + workers: 3 + gpus: 8 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 3072 + max-num-seqs: 512 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + all2all-backend: "flashinfer_nvlink_one_sided" + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "8192" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-1p4d-dep4-tp8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-1p4d-dep4-tp8-stp.yaml new file mode 100644 index 0000000000..13dac8502f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-1p4d-dep4-tp8-stp.yaml @@ -0,0 +1,100 @@ +schema: 2 +name: "kimi-vllm-disagg-gb300-1p4d-dep4-tp8" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.3.0.dev20260601 +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 4096 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + block-size: 64 + attention-backend: "FLASHINFER_MLA" + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.9 + + decode: + nodes: 8 + workers: 4 + gpus: 8 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + max-model-len: 10240 + max-num-seqs: 4 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + all2all-backend: "flashinfer_nvlink_one_sided" + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-1p8d-dep4-tp4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-1p8d-dep4-tp4-stp.yaml new file mode 100644 index 0000000000..4f550d98ce --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-1p8d-dep4-tp4-stp.yaml @@ -0,0 +1,100 @@ +schema: 2 +name: "kimi-vllm-disagg-gb300-1p8d-dep4-tp4" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.3.0.dev20260601 +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 4096 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + block-size: 64 + attention-backend: "FLASHINFER_MLA" + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.9 + + decode: + nodes: 8 + workers: 8 + gpus: 4 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 10240 + max-num-seqs: 768 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + all2all-backend: "flashinfer_nvlink_one_sided" + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "32x128" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-2p1d-dep4-dep24-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-2p1d-dep4-dep24-stp.yaml new file mode 100644 index 0000000000..290e65b595 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-2p1d-dep4-dep24-stp.yaml @@ -0,0 +1,103 @@ +schema: 2 +name: "kimi-vllm-disagg-gb300-2p1d-dep4-dep24" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.3.0.dev20260601 +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 4096 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + block-size: 64 + attention-backend: "FLASHINFER_MLA" + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.9 + + decode: + nodes: 6 + workers: 1 + gpus: 24 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 24 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 43 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + all2all-backend: "flashinfer_nvlink_one_sided" + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 128 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-3p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-3p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..0eec2c5d6d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-3p1d-dep4-dep16-stp.yaml @@ -0,0 +1,103 @@ +schema: 2 +name: "kimi-vllm-disagg-gb300-3p1d-dep4-dep16" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.3.0.dev20260601 +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 3 + workers: 3 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 4096 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + block-size: 64 + attention-backend: "FLASHINFER_MLA" + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.9 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 64 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + all2all-backend: "flashinfer_nvlink_one_sided" + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 128 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep8-stp.yaml new file mode 100644 index 0000000000..795d4b6a58 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep8-stp.yaml @@ -0,0 +1,104 @@ +schema: 2 +name: "kimi-vllm-disagg-gb300-4p1d-dep4-dep8" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.3.0.dev20260601 +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 4 + workers: 4 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 4096 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + block-size: 64 + attention-backend: "FLASHINFER_MLA" + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.9 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 256 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + all2all-backend: "flashinfer_nvlink_one_sided" + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 512 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "3072" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-8p1d-dep4-dep24-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-8p1d-dep4-dep24-stp.yaml new file mode 100644 index 0000000000..2b09c22b9a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-8p1d-dep4-dep24-stp.yaml @@ -0,0 +1,104 @@ +schema: 2 +name: "kimi-vllm-disagg-gb300-8p1d-dep4-dep24" + +model: + path: "kimi-k2.5-nvfp4" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp4" + +dynamo: + install: true + + source: + pypi: 1.3.0.dev20260601 +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 8 + workers: 8 + gpus: 4 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 4096 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + block-size: 64 + attention-backend: "FLASHINFER_MLA" + attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.9 + + decode: + nodes: 6 + workers: 1 + gpus: 24 + + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: "nvidia/Kimi-K2.5-NVFP4" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 24 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 512 + max-num-batched-tokens: 10240 + safetensors-load-strategy: "prefetch" + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-chunked-prefill: true + async-scheduling: true + attention-backend: "FLASHINFER_MLA" + block-size: 64 + all2all-backend: "flashinfer_nvlink_one_sided" + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 128 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "15360" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p1d-dep8-dep8-c2048-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p1d-dep8-dep8-c2048-stp.yaml new file mode 100644 index 0000000000..28760f9e8f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p1d-dep8-dep8-c2048-stp.yaml @@ -0,0 +1,135 @@ +schema: 2 +name: kimi-k2.6-vllm-disagg-b200-1p1d-dep8-dep8-c2048 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + install: true + source: + wheel: 1.3.0.dev20260721 +resources: + gpu_type: b200 + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + connector: + dp_launch_mode: per_gpu +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep8"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '2048' + req_rate: inf + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p1d-dep8-tp8-c1-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p1d-dep8-tp8-c1-stp.yaml new file mode 100644 index 0000000000..4012d76f89 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p1d-dep8-tp8-c1-stp.yaml @@ -0,0 +1,133 @@ +schema: 2 +name: kimi-k2.6-vllm-disagg-b200-1p1d-dep8-tp8-c1 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + install: true + source: + wheel: 1.3.0.dev20260721 +resources: + gpu_type: b200 + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + connector: + dp_launch_mode: per_gpu +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep8"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '0' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + max-model-len: 9216 + max-num-seqs: 1024 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 256 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '1' + req_rate: inf + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p4d-dep4-tp4-c512-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p4d-dep4-tp4-c512-stp.yaml new file mode 100644 index 0000000000..a269f42004 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p4d-dep4-tp4-c512-stp.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: kimi-k2.6-vllm-disagg-b200-1p4d-dep4-tp4-c512 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + install: true + source: + wheel: 1.3.0.dev20260721 +resources: + gpu_type: b200 + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + connector: + dp_launch_mode: per_gpu +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.94 + decode: + nodes: 2 + workers: 4 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 9216 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.93 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '512' + req_rate: inf + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p8d-dep4-tp4-c128-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p8d-dep4-tp4-c128-stp.yaml new file mode 100644 index 0000000000..b367309ebb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p8d-dep4-tp4-c128-stp.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: kimi-k2.6-vllm-disagg-b200-1p8d-dep4-tp4-c128 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + install: true + source: + wheel: 1.3.0.dev20260721 +resources: + gpu_type: b200 + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + connector: + dp_launch_mode: per_gpu +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.94 + decode: + nodes: 4 + workers: 8 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 9216 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.93 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '128' + req_rate: inf + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p8d-dep4-tp4-c32-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p8d-dep4-tp4-c32-stp.yaml new file mode 100644 index 0000000000..cb627bbd62 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p8d-dep4-tp4-c32-stp.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: kimi-k2.6-vllm-disagg-b200-1p8d-dep4-tp4-c32 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + install: true + source: + wheel: 1.3.0.dev20260721 +resources: + gpu_type: b200 + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + connector: + dp_launch_mode: per_gpu +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.93 + decode: + nodes: 4 + workers: 8 + gpus: 4 + env: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 9216 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.93 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '32' + req_rate: inf + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-2p1d-dep8-dep8-c8192-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-2p1d-dep8-dep8-c8192-stp.yaml new file mode 100644 index 0000000000..e054f0cee3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-2p1d-dep8-dep8-c8192-stp.yaml @@ -0,0 +1,135 @@ +schema: 2 +name: kimi-k2.6-vllm-disagg-b200-2p1d-dep8-dep8-c8192 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + install: true + source: + wheel: 1.3.0.dev20260721 +resources: + gpu_type: b200 + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + connector: + dp_launch_mode: per_gpu +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 8 + env: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '8192' + req_rate: inf + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml new file mode 100644 index 0000000000..532de487ee --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml @@ -0,0 +1,142 @@ +schema: 2 +name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c1-agentic" + +model: + path: "kimik3" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 +frontend: + type: vllm + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + agg: + nodes: 2 + workers: 1 + gpus: 16 + + env: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.92 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 2 + max-num-batched-tokens: 8192 + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' + prefix-match-unit: 128 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,64,128,256,512,1024,2048,4096,8192]}' + +services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + +benchmark: + type: custom + concurrencies: [1] + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml new file mode 100644 index 0000000000..590cfc7f7e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml @@ -0,0 +1,142 @@ +schema: 2 +name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c14-agentic" + +model: + path: "kimik3" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 +frontend: + type: vllm + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + agg: + nodes: 2 + workers: 1 + gpus: 16 + + env: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.92 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 28 + max-num-batched-tokens: 8192 + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' + prefix-match-unit: 128 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,40,56,64,80,104,128,256,512,1024,2048,4096,8192]}' + +services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + +benchmark: + type: custom + concurrencies: [14] + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml new file mode 100644 index 0000000000..cdcb3fb8d0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml @@ -0,0 +1,142 @@ +schema: 2 +name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c24-agentic" + +model: + path: "kimik3" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 +frontend: + type: vllm + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + agg: + nodes: 2 + workers: 1 + gpus: 16 + + env: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.92 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 48 + max-num-batched-tokens: 8192 + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' + prefix-match-unit: 128 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,40,56,64,80,104,128,256,512,1024,2048,4096,8192]}' + +services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + +benchmark: + type: custom + concurrencies: [24] + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml new file mode 100644 index 0000000000..a45f184bec --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml @@ -0,0 +1,142 @@ +schema: 2 +name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c4-agentic" + +model: + path: "kimik3" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 +frontend: + type: vllm + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + agg: + nodes: 2 + workers: 1 + gpus: 16 + + env: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.92 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 8 + max-num-batched-tokens: 8192 + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' + prefix-match-unit: 128 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,128,256,512,1024,2048,4096,8192]}' + +services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + +benchmark: + type: custom + concurrencies: [4] + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml new file mode 100644 index 0000000000..a8d44175ac --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml @@ -0,0 +1,142 @@ +schema: 2 +name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c48-agentic" + +model: + path: "kimik3" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 +frontend: + type: vllm + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + agg: + nodes: 2 + workers: 1 + gpus: 16 + + env: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.92 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 96 + max-num-batched-tokens: 8192 + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' + prefix-match-unit: 128 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,7,18,34,53,64,75,100,128,256,512,1024,2048,4096,8192]}' + +services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + +benchmark: + type: custom + concurrencies: [48] + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml new file mode 100644 index 0000000000..2bff214dd2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml @@ -0,0 +1,142 @@ +schema: 2 +name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c8-agentic" + +model: + path: "kimik3" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 +frontend: + type: vllm + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + agg: + nodes: 2 + workers: 1 + gpus: 16 + + env: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.92 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 16 + max-num-batched-tokens: 8192 + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' + prefix-match-unit: 128 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,40,56,64,80,104,128,256,512,1024,2048,4096,8192]}' + +services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + +benchmark: + type: custom + concurrencies: [8] + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml new file mode 100644 index 0000000000..28018a5910 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml @@ -0,0 +1,141 @@ +schema: 2 +name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c96-agentic" + +model: + path: "kimik3" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 +frontend: + type: vllm + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + agg: + nodes: 2 + workers: 1 + gpus: 16 + + env: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.92 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 192 + max-num-batched-tokens: 8192 + prefix-match-unit: 128 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,7,18,34,53,64,75,100,128,256,512,1024,2048,4096,8192]}' + +services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + +benchmark: + type: custom + concurrencies: [96] + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml new file mode 100644 index 0000000000..74e582273a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml @@ -0,0 +1,161 @@ +# GB200 TP16/DCP16 aggregate profile with DSpark K=4 and max-num-seqs 2. +schema: 2 +name: "kimi-k3-vllm-agg-gb200-dcp16-dspark4-maxseq2-mooncake-agentic" + +model: + path: "kimi-k3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + install: true + + source: + rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "96GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + enable-auto-tool-choice: true + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +engine: + type: vllm + connector: +roles: + agg: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Preserve the measured collective fallback when moving to the built image. + VLLM_USE_DIRECT_DCP_A2A: "0" + VLLM_USE_DIRECT_DCP_Q_GATHER: "0" + VLLM_USE_DIRECT_DCP_KV_GATHER: "0" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + decode-context-parallel-size: 16 + dcp-comm-backend: "a2a" + max-num-seqs: 2 + max-num-batched-tokens: 8192 + trust-remote-code: true + language-model-only: true + dyn-tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + dyn-reasoning-parser: "kimi_k3" + load-format: "safetensors" + safetensors-load-strategy: "lazy" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # Throughput jobs inject the committed K=4 golden AL (3.36); EVAL_ONLY + # preserves this real target-verification configuration. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + max-cudagraph-capture-size: 1024 + kv-cache-memory: 10737418240 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will + take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml new file mode 100644 index 0000000000..9e0c31f55d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml @@ -0,0 +1,157 @@ +# GB200 TP16/DCP16 aggregate profile without speculative decoding. +schema: 2 +name: "kimi-k3-vllm-agg-gb200-dcp16-nospec-mooncake-agentic" + +model: + path: "kimi-k3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + install: true + + source: + rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "96GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + enable-auto-tool-choice: true + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +engine: + type: vllm + connector: +roles: + agg: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Preserve the measured collective fallback when moving to the built image. + VLLM_USE_DIRECT_DCP_A2A: "0" + VLLM_USE_DIRECT_DCP_Q_GATHER: "0" + VLLM_USE_DIRECT_DCP_KV_GATHER: "0" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + decode-context-parallel-size: 16 + dcp-comm-backend: "a2a" + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + dyn-tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + dyn-reasoning-parser: "kimi_k3" + load-format: "safetensors" + safetensors-load-strategy: "lazy" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "FLASHINFER_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + max-cudagraph-capture-size: 1024 + max-num-seqs: 1000 + kv-cache-memory: 10737418240 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will + take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml new file mode 100644 index 0000000000..c7b7d8fe67 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml @@ -0,0 +1,163 @@ +schema: 2 +name: "kimi-k3-vllm-agg-gb200-dep16-throughput-vllm-simple-offload-agentic" + +# High-concurrency host-DRAM KV-offload variant of the official throughput- +# oriented multi_node_dep profile. TP4 x DP4 gives EP16 across four four-GPU +# GB200 nodes. Each TP rank receives a 128 GiB CPU KV pool (512 GiB per node). +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_dep + +model: + path: "kimi-k3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0" + +dynamo: + install: true + + source: + rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" +setup_script: kimik3-dspark-config-compat.sh + +environment: + ETCD_LEASE_TTL: "7200" + +slurm: + time_limit: "12:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + # Use vLLM's Kimi K3 parser so OpenAI responses expose structured + # tool_calls instead of raw XTML in message.content. + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + enable-auto-tool-choice: true + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-min-initial-workers: 1 + kv-cache-block-size: 64 + +engine: + type: vllm + connector: +roles: + agg: + nodes: 4 + workers: 1 + gpus: 16 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + NVIDIA_GDRCOPY: "1" + PYTHONHASHSEED: "42" + PYTORCH_ALLOC_CONF: "expandable_segments:True" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-dep16-offload-{job_id}" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + # FlashInfer's larger TP4 MoE representation leaves too little transient + # HBM for fastsafetensors' GPU-staging path when DSpark is loaded. + load-format: "safetensors" + safetensors-load-strategy: "lazy" + kv-cache-dtype: "fp8" + attention-backend: "FLASHINFER_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # DeepGEMM mega-MoE grid barriers time out under DSpark MRV2 with the + # one-sided all-to-all worker; use the supported Kimi K3 FlashInfer path. + moe-backend: "flashinfer_trtllm" + kda-prefill-backend: "flashkda" + kernel-config: '{"enable_cutedsl_warmup":true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.94 + # The largest offload point is c384 / DP4 = 96 sequences per engine. + # Capture even sequence counts: all configured DP4 steady-state batch + # sizes are exact hits, while odd loads pad by at most one sequence. + max-num-seqs: 96 + max-num-batched-tokens: 16384 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' + compilation-config: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96,102,108,114,120,126,132,138,144,150,156,162,168,174,180,186,192,198,204,210,216,222,228,234,240,246,252,258,264,270,276,282,288]}' + block-size: 64 + language-model-only: true + disable-custom-all-reduce: true + enable-prefix-caching: true + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' + scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" + dyn-tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + dyn-reasoning-parser: "kimi_k3" + no-enable-flashinfer-autotune: true + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AGENTIC_WARMUP_GRACE_PERIOD: "3600" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml new file mode 100644 index 0000000000..5dd6bce73e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml @@ -0,0 +1,160 @@ +schema: 2 +name: "kimi-k3-vllm-agg-gb200-dep16-throughput-agentic" + +# Day-0 GB200 translation of the official throughput-oriented multi_node_dep +# profile. TP4 x DP4 gives EP16 across four four-GPU GB200 nodes. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_dep + +model: + path: "kimi-k3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0" + +dynamo: + install: true + + source: + rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" +setup_script: kimik3-dspark-config-compat.sh + +environment: + ETCD_LEASE_TTL: "7200" + +slurm: + time_limit: "12:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + # Use vLLM's Kimi K3 parser so OpenAI responses expose structured + # tool_calls instead of raw XTML in message.content. + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + enable-auto-tool-choice: true + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-min-initial-workers: 1 + kv-cache-block-size: 64 + +engine: + type: vllm + connector: +roles: + agg: + nodes: 4 + workers: 1 + gpus: 16 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + NVIDIA_GDRCOPY: "1" + PYTORCH_ALLOC_CONF: "expandable_segments:True" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-dep16-{job_id}" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + # FlashInfer's larger TP4 MoE representation leaves too little transient + # HBM for fastsafetensors' GPU-staging path when DSpark is loaded. + load-format: "safetensors" + safetensors-load-strategy: "lazy" + kv-cache-dtype: "fp8" + attention-backend: "FLASHINFER_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # DeepGEMM mega-MoE grid barriers time out under DSpark MRV2 with the + # one-sided all-to-all worker; use the supported Kimi K3 FlashInfer path. + moe-backend: "flashinfer_trtllm" + kda-prefill-backend: "flashkda" + kernel-config: '{"enable_cutedsl_warmup":true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.94 + # The largest regular DEP point is c256 / DP4 = 64 sequences per engine. + # Capturing 128 sequence slots consumes 10.9 GiB and leaves too little + # runtime workspace for FlashInfer's MXFP4 MoE kernel. + max-num-seqs: 64 + max-num-batched-tokens: 16384 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' + compilation-config: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96,99,102,105,108,111,114,117,120,123,126,129,132,135,138,141,144,147,150,153,156,159,162,165,168,171,174,177,180,183,186,189,192]}' + block-size: 64 + language-model-only: true + disable-custom-all-reduce: true + enable-prefix-caching: true + scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" + dyn-tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + dyn-reasoning-parser: "kimi_k3" + no-enable-flashinfer-autotune: true + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AGENTIC_WARMUP_GRACE_PERIOD: "3600" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml new file mode 100644 index 0000000000..e4c10ce646 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml @@ -0,0 +1,151 @@ +schema: 2 +name: "kimi-k3-vllm-agg-gb200-tep16-balanced-agentic" + +# Day-0 GB200 translation of the official balanced multi_node_tep profile. +# Dense layers and MoE experts are sharded across 16 GPUs on four GB200 nodes +# with the official FP8 KV cache. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tep + +model: + path: "kimi-k3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0" + +dynamo: + install: true + + source: + rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" +setup_script: kimik3-dspark-config-compat.sh + +environment: + ETCD_LEASE_TTL: "7200" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + # Use vLLM's Kimi K3 parser so OpenAI responses expose structured + # tool_calls instead of raw XTML in message.content. + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + enable-auto-tool-choice: true + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-min-initial-workers: 1 + kv-cache-block-size: 64 + +engine: + type: vllm + connector: +roles: + agg: + nodes: 4 + workers: 1 + gpus: 16 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + NVIDIA_GDRCOPY: "1" + PYTORCH_ALLOC_CONF: "expandable_segments:True" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-tep16-{job_id}" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + pipeline-parallel-size: 1 + enable-expert-parallel: true + trust-remote-code: true + load-format: "fastsafetensors" + safetensors-load-strategy: "lazy" + kv-cache-dtype: "fp8" + attention-backend: "FLASHINFER_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + moe-backend: "flashinfer_trtllm" + kda-prefill-backend: "flashkda" + kernel-config: '{"enable_cutedsl_warmup":true}' + all2all-backend: "flashinfer_nvlink_one_sided" + gpu-memory-utilization: 0.92 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' + compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96],"pass_config":{"fuse_allreduce_rms":false}}' + block-size: 64 + language-model-only: true + disable-custom-all-reduce: true + enable-prefix-caching: true + scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" + dyn-tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + dyn-reasoning-parser: "kimi_k3" + no-enable-flashinfer-autotune: true + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml new file mode 100644 index 0000000000..63077310ec --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml @@ -0,0 +1,149 @@ +schema: 2 +name: "kimi-k3-vllm-agg-gb200-tp16-latency-agentic" + +# Day-0 GB200 translation of the official latency-oriented multi_node_tp +# profile. TP16 spans four GB200 nodes and uses the official FP8 KV cache. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tp + +model: + path: "kimi-k3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0" + +dynamo: + install: true + + source: + rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" +setup_script: kimik3-dspark-config-compat.sh + +environment: + ETCD_LEASE_TTL: "7200" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + args: + # Use vLLM's Kimi K3 parser so OpenAI responses expose structured + # tool_calls instead of raw XTML in message.content. + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + enable-auto-tool-choice: true + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-min-initial-workers: 1 + kv-cache-block-size: 64 + +engine: + type: vllm + connector: +roles: + agg: + nodes: 4 + workers: 1 + gpus: 16 + + env: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + NVIDIA_GDRCOPY: "1" + PYTORCH_ALLOC_CONF: "expandable_segments:True" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-tp16-{job_id}" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + pipeline-parallel-size: 1 + trust-remote-code: true + load-format: "fastsafetensors" + safetensors-load-strategy: "lazy" + kv-cache-dtype: "fp8" + attention-backend: "FLASHINFER_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + moe-backend: "flashinfer_trtllm" + kda-prefill-backend: "flashkda" + kernel-config: '{"enable_cutedsl_warmup":true}' + gpu-memory-utilization: 0.92 + max-num-seqs: 8 + max-num-batched-tokens: 8192 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' + compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24],"pass_config":{"fuse_allreduce_rms":false}}' + block-size: 64 + language-model-only: true + disable-custom-all-reduce: true + enable-prefix-caching: true + scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" + dyn-tool-call-parser: "kimi_k3" + reasoning-parser: "kimi_k3" + dyn-reasoning-parser: "kimi_k3" + no-enable-flashinfer-autotune: true + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml new file mode 100644 index 0000000000..72c326953c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml @@ -0,0 +1,130 @@ +schema: 2 +name: "kimik3-vllm-agg-gb200-tp8pp2-mooncake-c16-agentic" + +model: + path: "kimi-k3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: vllm + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + agg: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.92 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 32 + max-num-batched-tokens: 8192 + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' + prefix-match-unit: 128 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,40,56,64,80,104,128,256,512,1024,2048,4096,8192]}' + +services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml new file mode 100644 index 0000000000..8ca7c2e600 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml @@ -0,0 +1,130 @@ +schema: 2 +name: "kimik3-vllm-agg-gb200-tp8pp2-mooncake-c32-agentic" + +model: + path: "kimi-k3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: vllm + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + agg: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.92 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 64 + max-num-batched-tokens: 8192 + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' + prefix-match-unit: 128 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,7,18,34,53,64,75,100,128,256,512,1024,2048,4096,8192]}' + +services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml new file mode 100644 index 0000000000..476afb2b03 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml @@ -0,0 +1,129 @@ +schema: 2 +name: "kimik3-vllm-agg-gb200-tp8pp2-mooncake-c48-agentic" + +model: + path: "kimi-k3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: vllm + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + agg: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.92 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 96 + max-num-batched-tokens: 8192 + prefix-match-unit: 128 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,7,18,34,53,64,75,100,128,256,512,1024,2048,4096,8192]}' + +services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml new file mode 100644 index 0000000000..b304fcb735 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml @@ -0,0 +1,129 @@ +schema: 2 +name: "kimik3-vllm-agg-gb200-tp8pp2-mooncake-c72-agentic" + +model: + path: "kimi-k3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: vllm + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + agg: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.92 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 144 + max-num-batched-tokens: 8192 + prefix-match-unit: 128 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,7,18,34,53,64,75,100,128,256,512,1024,2048,4096,8192]}' + +services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml new file mode 100644 index 0000000000..6004f28c39 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml @@ -0,0 +1,129 @@ +schema: 2 +name: "kimik3-vllm-agg-gb200-tp8pp2-mooncake-c96-agentic" + +model: + path: "kimi-k3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: vllm + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + agg: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_RPC_TIMEOUT: "600000" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + decode-context-parallel-size: 8 + dcp-comm-backend: a2a + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.92 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 192 + max-num-batched-tokens: 8192 + prefix-match-unit: 128 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,7,18,34,53,64,75,100,128,256,512,1024,2048,4096,8192]}' + +services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml new file mode 100644 index 0000000000..3126cbeb25 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml @@ -0,0 +1,165 @@ +schema: 2 +name: "kimi-k3-vllm-agg-gb300-dcp8-dspark4-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + install: true + + source: + rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + args: + - "--default_kv_lease_ttl=60000" + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "160GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +engine: + type: vllm + connector: +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Use the direct DCP a2a/gather kernels rather than the collective + # fallback, matching the B300 arm (kimik3_fp4_b300_vllm_mtp.sh). + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + ETCD_LEASE_TTL: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_SLICE_SIZE: "1048576" + MC_MAX_MR_SIZE: "4294967296" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + gpu-memory-utilization: 0.92 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + enable-flashinfer-autotune: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + mamba-ssm-cache-dtype: "bfloat16" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + max-cudagraph-capture-size: 1024 + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will + take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 + +benchmark: + type: custom + concurrencies: [48, 52, 56] + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + placement: + node: head diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml new file mode 100644 index 0000000000..36f434b7b8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml @@ -0,0 +1,159 @@ +schema: 2 +name: "kimi-k3-vllm-agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + install: true + + source: + rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + args: + - "--default_kv_lease_ttl=60000" + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "160GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +engine: + type: vllm + connector: +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + ETCD_LEASE_TTL: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + MC_SLICE_SIZE: "1048576" + MC_MAX_MR_SIZE: "4294967296" + WITH_NVIDIA_PEERMEM: "0" + VLLM_LOG_STATS_INTERVAL: "1" + + args: + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' + served-model-name: "moonshotai/Kimi-K3" + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + mamba-ssm-cache-dtype: "bfloat16" + gpu-memory-utilization: 0.92 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + max-num-seqs: 2 + max-num-batched-tokens: 8192 + trust-remote-code: true + max-cudagraph-capture-size: 1024 + stream-interval: 10 + language-model-only: true + moe-backend: "auto" + enable-flashinfer-autotune: true + enable-cumem-allocator: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block"}' + enable-prefix-caching: true + prefix-match-unit: 128 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will + take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 + +benchmark: + type: custom + concurrencies: [1, 4] + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + placement: + node: head diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml new file mode 100644 index 0000000000..3833ea7c83 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml @@ -0,0 +1,188 @@ +schema: 2 +name: "kimi-k3-vllm-disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + install: true + + source: + rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + args: + - "--default_kv_lease_ttl=60000" + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "160GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: &kimi_env + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" + DYN_REQUEST_PLANE: "tcp" + ETCD_LEASE_TTL: "600" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + MC_SLICE_SIZE: "1048576" + MC_MAX_MR_SIZE: "4294967296" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + WITH_NVIDIA_PEERMEM: "0" + NCCL_NET_PLUGIN: "none" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_TCP_AF_PRIO: "inet" + VLLM_SSM_CONV_STATE_LAYOUT: "DS" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "moonshotai/Kimi-K3" + prefix-match-unit: 128 + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + mamba-ssm-cache-dtype: "bfloat16" + gpu-memory-utilization: 0.92 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + cp-kv-cache-interleave-size: 1 + dcp-comm-backend: "a2a" + enable-flashinfer-autotune: true + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true + kv_events: true + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + <<: *kimi_env + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "moonshotai/Kimi-K3" + prefix-match-unit: 128 + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + mamba-ssm-cache-dtype: "bfloat16" + gpu-memory-utilization: 0.92 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + cp-kv-cache-interleave-size: 1 + dcp-comm-backend: "a2a" + enable-flashinfer-autotune: true + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will + take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml new file mode 100644 index 0000000000..934327431c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml @@ -0,0 +1,188 @@ +schema: 2 +name: "kimi-k3-vllm-disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + install: true + + source: + rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + args: + - "--default_kv_lease_ttl=60000" + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "160GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: &kimi_env + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" + DYN_REQUEST_PLANE: "tcp" + ETCD_LEASE_TTL: "600" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + MC_SLICE_SIZE: "1048576" + MC_MAX_MR_SIZE: "4294967296" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + WITH_NVIDIA_PEERMEM: "0" + NCCL_NET_PLUGIN: "none" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_TCP_AF_PRIO: "inet" + VLLM_SSM_CONV_STATE_LAYOUT: "DS" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "moonshotai/Kimi-K3" + prefix-match-unit: 128 + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + mamba-ssm-cache-dtype: "bfloat16" + gpu-memory-utilization: 0.92 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + cp-kv-cache-interleave-size: 1 + dcp-comm-backend: "a2a" + enable-flashinfer-autotune: true + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true + kv_events: true + decode: + nodes: 4 + workers: 2 + gpus: 8 + + env: + <<: *kimi_env + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "moonshotai/Kimi-K3" + prefix-match-unit: 128 + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + mamba-ssm-cache-dtype: "bfloat16" + gpu-memory-utilization: 0.92 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + cp-kv-cache-interleave-size: 1 + dcp-comm-backend: "a2a" + enable-flashinfer-autotune: true + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will + take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml new file mode 100644 index 0000000000..3d16c4edb3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml @@ -0,0 +1,188 @@ +schema: 2 +name: "kimi-k3-vllm-disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + install: true + + source: + rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + args: + - "--default_kv_lease_ttl=60000" + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "160GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: &kimi_env + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" + DYN_REQUEST_PLANE: "tcp" + ETCD_LEASE_TTL: "600" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + MC_SLICE_SIZE: "1048576" + MC_MAX_MR_SIZE: "4294967296" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + WITH_NVIDIA_PEERMEM: "0" + NCCL_NET_PLUGIN: "none" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_TCP_AF_PRIO: "inet" + VLLM_SSM_CONV_STATE_LAYOUT: "DS" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "moonshotai/Kimi-K3" + prefix-match-unit: 128 + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + mamba-ssm-cache-dtype: "bfloat16" + gpu-memory-utilization: 0.92 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + cp-kv-cache-interleave-size: 1 + dcp-comm-backend: "a2a" + enable-flashinfer-autotune: true + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true + kv_events: true + decode: + nodes: 6 + workers: 3 + gpus: 8 + + env: + <<: *kimi_env + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "moonshotai/Kimi-K3" + prefix-match-unit: 128 + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + mamba-ssm-cache-dtype: "bfloat16" + gpu-memory-utilization: 0.92 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + cp-kv-cache-interleave-size: 1 + dcp-comm-backend: "a2a" + enable-flashinfer-autotune: true + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' + enable-prefix-caching: true + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will + take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml new file mode 100644 index 0000000000..72cd83e6c3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml @@ -0,0 +1,188 @@ +schema: 2 +name: "kimi-k3-vllm-disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + install: true + + source: + rev: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + args: + - "--default_kv_lease_ttl=60000" + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "160GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: &kimi_env + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" + DYN_REQUEST_PLANE: "tcp" + ETCD_LEASE_TTL: "600" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + MC_SLICE_SIZE: "1048576" + MC_MAX_MR_SIZE: "4294967296" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + WITH_NVIDIA_PEERMEM: "0" + NCCL_NET_PLUGIN: "none" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_TCP_AF_PRIO: "inet" + VLLM_SSM_CONV_STATE_LAYOUT: "DS" + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "moonshotai/Kimi-K3" + prefix-match-unit: 128 + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + mamba-ssm-cache-dtype: "bfloat16" + gpu-memory-utilization: 0.92 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + cp-kv-cache-interleave-size: 1 + dcp-comm-backend: "a2a" + enable-flashinfer-autotune: true + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block"}' + enable-prefix-caching: true + kv_events: true + decode: + nodes: 6 + workers: 3 + gpus: 8 + + env: + <<: *kimi_env + args: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "moonshotai/Kimi-K3" + prefix-match-unit: 128 + load-format: "fastsafetensors" + kv-cache-dtype: "fp8" + mamba-ssm-cache-dtype: "bfloat16" + gpu-memory-utilization: 0.92 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + cp-kv-cache-interleave-size: 1 + dcp-comm-backend: "a2a" + enable-flashinfer-autotune: true + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block"}' + enable-prefix-caching: true + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will + take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml new file mode 100644 index 0000000000..eac0fc430d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: "kimik3-vllm-agg-h200-tp16dp2ep32-latency-agentic" + +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + vllm: "0.1.dev19262+gb6bbf29dd.d20260727" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "h200" + gpus_per_node: 8 +frontend: + type: vllm + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + agg: + nodes: 4 + workers: 1 + gpus: 32 + + env: + CUDA_LAUNCH_BLOCKING: "1" + GLOO_SOCKET_IFNAME: "eth0" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + NCCL_SOCKET_IFNAME: "eth0" + NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + PYTHONNOUSERSITE: "1" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_ROUTED_DOWN_PROJ_STREAM_TOKEN_THRESHOLD: "0" + VLLM_USE_V2_MODEL_RUNNER: "1" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + data-parallel-size: 2 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: marlin + attention-backend: FLASHMLA + gpu-memory-utilization: 0.975 + max-num-seqs: 5 + max-num-batched-tokens: 4096 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' + enforce-eager: true + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + no-enable-flashinfer-autotune: true + disable-custom-all-reduce: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + +srun_options: + container-remap-root: "" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + +benchmark: + type: custom + concurrencies: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12] + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml new file mode 100644 index 0000000000..77db5a7617 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml @@ -0,0 +1,97 @@ +schema: 2 +name: "kimik3-vllm-agg-h200-tp8dp4ep32-balanced-agentic" + +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + vllm: "0.1.dev19262+gb6bbf29dd.d20260727" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "h200" + gpus_per_node: 8 +frontend: + type: vllm + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + agg: + nodes: 4 + workers: 1 + gpus: 32 + + env: + CUDA_LAUNCH_BLOCKING: "1" + GLOO_SOCKET_IFNAME: "eth0" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + NCCL_SOCKET_IFNAME: "eth0" + NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + PYTHONNOUSERSITE: "1" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_ROUTED_DOWN_PROJ_STREAM_TOKEN_THRESHOLD: "0" + VLLM_USE_V2_MODEL_RUNNER: "1" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + data-parallel-size: 4 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: marlin + attention-backend: FLASHMLA + gpu-memory-utilization: 0.95 + max-num-seqs: 8 + max-num-batched-tokens: 4096 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' + enforce-eager: true + no-async-scheduling: true + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + no-enable-flashinfer-autotune: true + disable-custom-all-reduce: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml new file mode 100644 index 0000000000..9f64a4aec8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml @@ -0,0 +1,98 @@ +schema: 2 +name: "kimik3-vllm-agg-h200-tp8dp4ep32-vllm-simple-agentic" + +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + vllm: "0.1.dev19262+gb6bbf29dd.d20260727" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "h200" + gpus_per_node: 8 +frontend: + type: vllm + enable_multiple_frontends: false + +engine: + type: vllm + connector: +roles: + agg: + nodes: 4 + workers: 1 + gpus: 32 + + env: + CUDA_LAUNCH_BLOCKING: "1" + GLOO_SOCKET_IFNAME: "eth0" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + NCCL_SOCKET_IFNAME: "eth0" + NCCL_CUMEM_ENABLE: "1" + PYTHONHASHSEED: "42" + PYTHONNOUSERSITE: "1" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_ROUTED_DOWN_PROJ_STREAM_TOKEN_THRESHOLD: "0" + VLLM_USE_V2_MODEL_RUNNER: "1" + args: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + data-parallel-size: 4 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: marlin + attention-backend: FLASHMLA + gpu-memory-utilization: 0.95 + max-num-seqs: 16 + max-num-batched-tokens: 4096 + speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' + enforce-eager: true + no-async-scheduling: true + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + enable-prefix-caching: true + enable-prompt-tokens-details: true + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":154250000000,"lazy_offload":false}}' + no-enable-flashinfer-autotune: true + disable-custom-all-reduce: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml new file mode 100644 index 0000000000..cb65c6a1cb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml @@ -0,0 +1,152 @@ +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=10 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# Forced acceptance is NOT set here: runners/inject_synthetic_acceptance.py injects TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +schema: 2 +name: dynamo-agg-gb200-tp4-c10-b1-eagle3 +model: + path: minimax-m3-nvfp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 + precision: fp4 +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + request_plane: tcp +health_check: + max_attempts: 270 + interval_seconds: 10 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: + type: trtllm + numa_memory_bind: true + served_model_name: nvidia/MiniMax-M3-NVFP4 + publish_events_and_metrics: false +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + HF_HUB_CACHE: /hf_hub_cache + TQDM_DISABLE: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + PYTHONNOUSERSITE: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + args: + max_seq_len: 1048576 + max_num_tokens: 16384 + max_batch_size: 10 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + torch_compile_config: + enable_fullgraph: true + enable_inductor: false + enable_piecewise_cuda_graph: true + capture_num_tokens: + - 1 + - 512 + - 1024 + - 2048 + enable_userbuffers: true + max_num_streams: 3 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + free_gpu_memory_fraction: 0.94 + enable_block_reuse: true + block_reuse_policy: per_conversation + tokens_per_block: 128 + use_kv_cache_manager_v2: true + dtype: fp8 + event_buffer_max_size: 0 + host_cache_size: 137438953472 + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + enable_chunked_prefill: true + enable_autotuner: true + trust_remote_code: true + stream_interval: 20 + print_iter_log: true + num_postprocess_workers: 8 + enable_attention_dp: false + tensor_parallel_size: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + KV_OFFLOAD_BACKEND: native + KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' + TOTAL_CPU_DRAM_GB: '550' + MODEL: nvidia/MiniMax-M3-NVFP4 + MODEL_PREFIX: minimaxm3 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '10' + DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml new file mode 100644 index 0000000000..733a670fd1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml @@ -0,0 +1,157 @@ +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=15 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# Forced acceptance is NOT set here: runners/inject_synthetic_acceptance.py injects TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +schema: 2 +name: dynamo-agg-gb200-tp4-c15-b1-eagle3 +model: + path: minimax-m3-nvfp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 + precision: fp4 +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + request_plane: tcp +health_check: + max_attempts: 270 + interval_seconds: 10 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: + type: trtllm + numa_memory_bind: true + served_model_name: nvidia/MiniMax-M3-NVFP4 + publish_events_and_metrics: false +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + HF_HUB_CACHE: /hf_hub_cache + TQDM_DISABLE: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + PYTHONNOUSERSITE: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + args: + max_seq_len: 1048576 + max_num_tokens: 16384 + max_batch_size: 15 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + torch_compile_config: + enable_fullgraph: true + enable_inductor: false + enable_piecewise_cuda_graph: true + capture_num_tokens: + - 1 + - 512 + - 1024 + - 2048 + enable_userbuffers: true + max_num_streams: 3 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + free_gpu_memory_fraction: 0.94 + enable_block_reuse: true + block_reuse_policy: per_conversation + tokens_per_block: 128 + use_kv_cache_manager_v2: true + dtype: fp8 + event_buffer_max_size: 0 + host_cache_size: 137438953472 + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + enable_chunked_prefill: true + enable_autotuner: true + trust_remote_code: true + stream_interval: 20 + print_iter_log: true + num_postprocess_workers: 8 + enable_attention_dp: false + tensor_parallel_size: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + KV_OFFLOAD_BACKEND: native + KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' + TOTAL_CPU_DRAM_GB: '550' + MODEL: nvidia/MiniMax-M3-NVFP4 + MODEL_PREFIX: minimaxm3 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '15' + DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml new file mode 100644 index 0000000000..81ef85c077 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml @@ -0,0 +1,162 @@ +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=20 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# Forced acceptance is NOT set here: runners/inject_synthetic_acceptance.py injects TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +schema: 2 +name: dynamo-agg-gb200-tp4-c20-b1-eagle3 +model: + path: minimax-m3-nvfp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 + precision: fp4 +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + request_plane: tcp +health_check: + max_attempts: 270 + interval_seconds: 10 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: + type: trtllm + numa_memory_bind: true + served_model_name: nvidia/MiniMax-M3-NVFP4 + publish_events_and_metrics: false +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + HF_HUB_CACHE: /hf_hub_cache + TQDM_DISABLE: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + PYTHONNOUSERSITE: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + args: + max_seq_len: 1048576 + max_num_tokens: 16384 + max_batch_size: 20 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 16 + - 17 + - 18 + - 19 + - 20 + torch_compile_config: + enable_fullgraph: true + enable_inductor: false + enable_piecewise_cuda_graph: true + capture_num_tokens: + - 1 + - 512 + - 1024 + - 2048 + enable_userbuffers: true + max_num_streams: 3 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + free_gpu_memory_fraction: 0.94 + enable_block_reuse: true + block_reuse_policy: per_conversation + tokens_per_block: 128 + use_kv_cache_manager_v2: true + dtype: fp8 + event_buffer_max_size: 0 + host_cache_size: 137438953472 + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + enable_chunked_prefill: true + enable_autotuner: true + trust_remote_code: true + stream_interval: 20 + print_iter_log: true + num_postprocess_workers: 8 + enable_attention_dp: false + tensor_parallel_size: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + KV_OFFLOAD_BACKEND: native + KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' + TOTAL_CPU_DRAM_GB: '550' + MODEL: nvidia/MiniMax-M3-NVFP4 + MODEL_PREFIX: minimaxm3 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '20' + DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml new file mode 100644 index 0000000000..9373ea6b5e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml @@ -0,0 +1,162 @@ +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=25 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# Forced acceptance is NOT set here: runners/inject_synthetic_acceptance.py injects TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +schema: 2 +name: dynamo-agg-gb200-tp4-c25-b1-eagle3 +model: + path: minimax-m3-nvfp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 + precision: fp4 +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + request_plane: tcp +health_check: + max_attempts: 270 + interval_seconds: 10 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: + type: trtllm + numa_memory_bind: true + served_model_name: nvidia/MiniMax-M3-NVFP4 + publish_events_and_metrics: false +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + HF_HUB_CACHE: /hf_hub_cache + TQDM_DISABLE: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + PYTHONNOUSERSITE: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + args: + max_seq_len: 1048576 + max_num_tokens: 16384 + max_batch_size: 25 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 13 + - 14 + - 15 + - 17 + - 19 + - 21 + - 23 + - 25 + torch_compile_config: + enable_fullgraph: true + enable_inductor: false + enable_piecewise_cuda_graph: true + capture_num_tokens: + - 1 + - 512 + - 1024 + - 2048 + enable_userbuffers: true + max_num_streams: 3 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + free_gpu_memory_fraction: 0.94 + enable_block_reuse: true + block_reuse_policy: per_conversation + tokens_per_block: 128 + use_kv_cache_manager_v2: true + dtype: fp8 + event_buffer_max_size: 0 + host_cache_size: 137438953472 + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + enable_chunked_prefill: true + enable_autotuner: true + trust_remote_code: true + stream_interval: 20 + print_iter_log: true + num_postprocess_workers: 8 + enable_attention_dp: false + tensor_parallel_size: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + KV_OFFLOAD_BACKEND: native + KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' + TOTAL_CPU_DRAM_GB: '550' + MODEL: nvidia/MiniMax-M3-NVFP4 + MODEL_PREFIX: minimaxm3 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '25' + DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml new file mode 100644 index 0000000000..2028716931 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml @@ -0,0 +1,162 @@ +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=30 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# Forced acceptance is NOT set here: runners/inject_synthetic_acceptance.py injects TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +schema: 2 +name: dynamo-agg-gb200-tp4-c30-b1-eagle3 +model: + path: minimax-m3-nvfp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 + precision: fp4 +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + request_plane: tcp +health_check: + max_attempts: 270 + interval_seconds: 10 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: + type: trtllm + numa_memory_bind: true + served_model_name: nvidia/MiniMax-M3-NVFP4 + publish_events_and_metrics: false +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + HF_HUB_CACHE: /hf_hub_cache + TQDM_DISABLE: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + PYTHONNOUSERSITE: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + args: + max_seq_len: 1048576 + max_num_tokens: 16384 + max_batch_size: 30 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 11 + - 12 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 27 + - 30 + torch_compile_config: + enable_fullgraph: true + enable_inductor: false + enable_piecewise_cuda_graph: true + capture_num_tokens: + - 1 + - 512 + - 1024 + - 2048 + enable_userbuffers: true + max_num_streams: 3 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + free_gpu_memory_fraction: 0.94 + enable_block_reuse: true + block_reuse_policy: per_conversation + tokens_per_block: 128 + use_kv_cache_manager_v2: true + dtype: fp8 + event_buffer_max_size: 0 + host_cache_size: 137438953472 + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + enable_chunked_prefill: true + enable_autotuner: true + trust_remote_code: true + stream_interval: 20 + print_iter_log: true + num_postprocess_workers: 8 + enable_attention_dp: false + tensor_parallel_size: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + KV_OFFLOAD_BACKEND: native + KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' + TOTAL_CPU_DRAM_GB: '550' + MODEL: nvidia/MiniMax-M3-NVFP4 + MODEL_PREFIX: minimaxm3 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '30' + DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml new file mode 100644 index 0000000000..69921ca093 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml @@ -0,0 +1,166 @@ +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=40 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# Forced acceptance is NOT set here: runners/inject_synthetic_acceptance.py injects TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +schema: 2 +name: dynamo-agg-gb200-tp4-c40-b1-eagle3 +model: + path: minimax-m3-nvfp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 + precision: fp4 +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + request_plane: tcp +health_check: + max_attempts: 270 + interval_seconds: 10 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: + type: trtllm + numa_memory_bind: true + served_model_name: nvidia/MiniMax-M3-NVFP4 + publish_events_and_metrics: false +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + HF_HUB_CACHE: /hf_hub_cache + TQDM_DISABLE: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + PYTHONNOUSERSITE: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + args: + max_seq_len: 1048576 + max_num_tokens: 16384 + max_batch_size: 40 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + - 14 + - 16 + - 18 + - 20 + - 22 + - 24 + - 26 + - 28 + - 30 + - 32 + - 34 + - 36 + - 38 + - 40 + torch_compile_config: + enable_fullgraph: true + enable_inductor: false + enable_piecewise_cuda_graph: true + capture_num_tokens: + - 1 + - 512 + - 1024 + - 2048 + enable_userbuffers: true + max_num_streams: 3 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + free_gpu_memory_fraction: 0.94 + enable_block_reuse: true + block_reuse_policy: per_conversation + tokens_per_block: 128 + use_kv_cache_manager_v2: true + dtype: fp8 + event_buffer_max_size: 0 + host_cache_size: 137438953472 + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + enable_chunked_prefill: true + enable_autotuner: true + trust_remote_code: true + stream_interval: 20 + print_iter_log: true + num_postprocess_workers: 8 + enable_attention_dp: false + tensor_parallel_size: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + KV_OFFLOAD_BACKEND: native + KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' + TOTAL_CPU_DRAM_GB: '550' + MODEL: nvidia/MiniMax-M3-NVFP4 + MODEL_PREFIX: minimaxm3 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '40' + DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml new file mode 100644 index 0000000000..714d570785 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml @@ -0,0 +1,147 @@ +# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. +# Engine config is the max_batch_size=5 member of the TP4 recipe set. +# Host KV cache is set to 128 GiB per rank for the four-rank layout. +# Forced acceptance is NOT set here: runners/inject_synthetic_acceptance.py injects TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS +# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. +schema: 2 +name: dynamo-agg-gb200-tp4-c5-b1-eagle3 +model: + path: minimax-m3-nvfp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 + precision: fp4 +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + request_plane: tcp +health_check: + max_attempts: 270 + interval_seconds: 10 +resources: + gpu_type: gb200 + gpus_per_node: 4 +engine: + type: trtllm + numa_memory_bind: true + served_model_name: nvidia/MiniMax-M3-NVFP4 + publish_events_and_metrics: false +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + HF_HUB_OFFLINE: '1' + TRANSFORMERS_OFFLINE: '1' + HF_HUB_CACHE: /hf_hub_cache + TQDM_DISABLE: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PROFILE_LOG_RANKS: all + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + PYTHONNOUSERSITE: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + args: + max_seq_len: 1048576 + max_num_tokens: 16384 + max_batch_size: 5 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 3 + - 4 + - 5 + torch_compile_config: + enable_fullgraph: true + enable_inductor: false + enable_piecewise_cuda_graph: true + capture_num_tokens: + - 1 + - 512 + - 1024 + - 2048 + enable_userbuffers: true + max_num_streams: 3 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + free_gpu_memory_fraction: 0.94 + enable_block_reuse: true + block_reuse_policy: per_conversation + tokens_per_block: 128 + use_kv_cache_manager_v2: true + dtype: fp8 + event_buffer_max_size: 0 + host_cache_size: 137438953472 + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + enable_chunked_prefill: true + enable_autotuner: true + trust_remote_code: true + stream_interval: 20 + print_iter_log: true + num_postprocess_workers: 8 + enable_attention_dp: false + tensor_parallel_size: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram + KV_OFFLOAD_BACKEND: native + KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' + TOTAL_CPU_DRAM_GB: '550' + MODEL: nvidia/MiniMax-M3-NVFP4 + MODEL_PREFIX: minimaxm3 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '5' + DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p1d-dep2-tp4-c1-c16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p1d-dep2-tp4-c1-c16-stp.yaml new file mode 100644 index 0000000000..dab129059a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p1d-dep2-tp4-c1-c16-stp.yaml @@ -0,0 +1,66 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b200-1p1d-fp4-dep2-tp4-8k1k" +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" +resources: + gpu_type: "b200" + gpus_per_node: 8 +dynamo: {install: true, source: {pypi: 1.3.0.dev20260710}} +frontend: {type: dynamo, enable_multiple_frontends: false} +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: {VLLM_FLOAT32_MATMUL_PRECISION: high, UCX_TCP_AF_PRIO: "inet"} + args: + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + decode: + nodes: 1 + workers: 1 + gpus: 4 + env: {VLLM_FLOAT32_MATMUL_PRECISION: high, UCX_TCP_AF_PRIO: "inet"} + args: + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 +health_check: {max_attempts: 360, interval_seconds: 10} +benchmark: {type: "sa-bench", isl: 8192, osl: 1024, random_range_ratio: 0.8, concurrencies: "1x4x8x16", req_rate: "inf"} diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p1d-tp4-dep4-eagle3-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p1d-tp4-dep4-eagle3-mtp.yaml new file mode 100644 index 0000000000..0cc5e52ea0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p1d-tp4-dep4-eagle3-mtp.yaml @@ -0,0 +1,102 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b200-1p1d-tp4-dep4-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: fp4 + +resources: + gpu_type: b200 + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + allow_prefill_decode_colocation_across_nodes: true + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: colocate + workers: 1 + gpus: 4 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + UCX_TLS: cuda_copy,cuda_ipc,rc + VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: cutlass + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "256x512" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p1d-tp4-tp4-eagle3-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p1d-tp4-tp4-eagle3-mtp.yaml new file mode 100644 index 0000000000..f445cbb18d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p1d-tp4-tp4-eagle3-mtp.yaml @@ -0,0 +1,95 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b200-1p1d-tp4-tp4-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: fp4 + +resources: + gpu_type: b200 + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + allow_prefill_decode_colocation_across_nodes: true + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: colocate + workers: 1 + gpus: 4 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: cutlass + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "192x256x512" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p2d-dep2-tp4-c64-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p2d-dep2-tp4-c64-stp.yaml new file mode 100644 index 0000000000..b244a10580 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p2d-dep2-tp4-c64-stp.yaml @@ -0,0 +1,129 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b200-1p2d-fp4-dep2-tp4-c64-8k1k" +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" +resources: + gpu_type: "b200" + spread_workers: true + gpus_per_node: 8 +dynamo: {install: true, source: {pypi: 1.3.0.dev20260710}} +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: vllm + reasoning-parser: minimax_m3 +engine: + type: vllm + connector: +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TCP_AF_PRIO: "inet" + args: + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + enable-expert-parallel: true + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + max-model-len: 9472 + max-num-seqs: 16 + enforce-eager: true + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + no-enable-chunked-prefill: true + kv-cache-dtype: fp8 + async-scheduling: true + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + decode: + nodes: 2 + workers: 2 + gpus: 4 + env: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_MOE_DP_CHUNK_SIZE: '384' + VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: '8192' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TCP_AF_PRIO: "inet" + args: + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + max-model-len: 9472 + max-num-seqs: 32 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+rms_norm"],"pass_config":{}}' + kv-cache-dtype: fp8 + all2all-backend: deepep_low_latency + async-scheduling: true + stream-interval: 32 + enable-dbo: true + dbo-decode-token-threshold: 32 + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + block-size: 128 + language-model-only: true +health_check: {max_attempts: 360, interval_seconds: 10} +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "64" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p2d-tp4-tp4-eagle3-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p2d-tp4-tp4-eagle3-mtp.yaml new file mode 100644 index 0000000000..233efa0611 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p2d-tp4-tp4-eagle3-mtp.yaml @@ -0,0 +1,95 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b200-1p2d-tp4-tp4-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: fp4 + +resources: + gpu_type: b200 + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + allow_prefill_decode_colocation_across_nodes: true + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 1 + workers: 2 + gpus: 4 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: cutlass + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "32x64x96" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p3d-tp4-tp4-eagle3-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p3d-tp4-tp4-eagle3-mtp.yaml new file mode 100644 index 0000000000..d8fb3341e6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p3d-tp4-tp4-eagle3-mtp.yaml @@ -0,0 +1,95 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b200-1p3d-tp4-tp4-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: fp4 + +resources: + gpu_type: b200 + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + allow_prefill_decode_colocation_across_nodes: true + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 1 + workers: 3 + gpus: 4 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: cutlass + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "192" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p4d-tp4-tp4-eagle3-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p4d-tp4-tp4-eagle3-mtp.yaml new file mode 100644 index 0000000000..592b8c7e8c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p4d-tp4-tp4-eagle3-mtp.yaml @@ -0,0 +1,93 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b200-1p4d-tp4-tp4-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: fp4 + +resources: + gpu_type: b200 + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + allow_prefill_decode_colocation_across_nodes: true + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 4 + gpus: 4 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "4x8x16x32x64" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p6d-tp4-tp4-eagle3-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p6d-tp4-tp4-eagle3-mtp.yaml new file mode 100644 index 0000000000..78de49470f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p6d-tp4-tp4-eagle3-mtp.yaml @@ -0,0 +1,95 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b200-1p6d-tp4-tp4-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: fp4 + +resources: + gpu_type: b200 + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + allow_prefill_decode_colocation_across_nodes: true + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 3 + workers: 6 + gpus: 4 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: cutlass + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "12x24x48" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-2p2d-dep2-tp4-c128-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-2p2d-dep2-tp4-c128-stp.yaml new file mode 100644 index 0000000000..3bd83bfca6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-2p2d-dep2-tp4-c128-stp.yaml @@ -0,0 +1,129 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b200-2p2d-fp4-dep2-tp4-c128-8k1k" +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" +resources: + gpu_type: "b200" + spread_workers: true + gpus_per_node: 8 +dynamo: {install: true, source: {pypi: 1.3.0.dev20260710}} +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: vllm + reasoning-parser: minimax_m3 +engine: + type: vllm + connector: +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 2 + env: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TCP_AF_PRIO: "inet" + args: + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + enable-expert-parallel: true + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + max-model-len: 9472 + max-num-seqs: 16 + enforce-eager: true + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + no-enable-chunked-prefill: true + kv-cache-dtype: fp8 + async-scheduling: true + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + decode: + nodes: 2 + workers: 2 + gpus: 4 + env: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_MOE_DP_CHUNK_SIZE: '384' + VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: '8192' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TCP_AF_PRIO: "inet" + args: + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + max-model-len: 9472 + max-num-seqs: 64 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+rms_norm"],"pass_config":{}}' + kv-cache-dtype: fp8 + all2all-backend: deepep_low_latency + async-scheduling: true + stream-interval: 32 + enable-dbo: true + dbo-decode-token-threshold: 32 + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + block-size: 128 + language-model-only: true +health_check: {max_attempts: 360, interval_seconds: 10} +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "128" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-3p2d-dep2-tp4-c256-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-3p2d-dep2-tp4-c256-stp.yaml new file mode 100644 index 0000000000..55cf0e0856 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-3p2d-dep2-tp4-c256-stp.yaml @@ -0,0 +1,129 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b200-3p2d-fp4-dep2-tp4-c256-8k1k" +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" +resources: + gpu_type: "b200" + spread_workers: true + gpus_per_node: 8 +dynamo: {install: true, source: {pypi: 1.3.0.dev20260710}} +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: vllm + reasoning-parser: minimax_m3 +engine: + type: vllm + connector: +roles: + prefill: + nodes: 3 + workers: 3 + gpus: 2 + env: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TCP_AF_PRIO: "inet" + args: + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + enable-expert-parallel: true + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + max-model-len: 9472 + max-num-seqs: 16 + enforce-eager: true + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + no-enable-chunked-prefill: true + kv-cache-dtype: fp8 + async-scheduling: true + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + decode: + nodes: 2 + workers: 2 + gpus: 4 + env: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_MOE_DP_CHUNK_SIZE: '384' + VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: '8192' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TCP_AF_PRIO: "inet" + args: + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + max-model-len: 9472 + max-num-seqs: 128 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+rms_norm"],"pass_config":{}}' + kv-cache-dtype: fp8 + all2all-backend: deepep_low_latency + async-scheduling: true + stream-interval: 32 + enable-dbo: true + dbo-decode-token-threshold: 32 + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + block-size: 128 + language-model-only: true +health_check: {max_attempts: 360, interval_seconds: 10} +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "256" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-3p2d-dep2-tp4-c512-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-3p2d-dep2-tp4-c512-stp.yaml new file mode 100644 index 0000000000..0fe9dc378b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-3p2d-dep2-tp4-c512-stp.yaml @@ -0,0 +1,129 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b200-3p2d-fp4-dep2-tp4-c512-8k1k" +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" +resources: + gpu_type: "b200" + spread_workers: true + gpus_per_node: 8 +dynamo: {install: true, source: {pypi: 1.3.0.dev20260710}} +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: vllm + reasoning-parser: minimax_m3 +engine: + type: vllm + connector: +roles: + prefill: + nodes: 3 + workers: 3 + gpus: 2 + env: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TCP_AF_PRIO: "inet" + args: + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + enable-expert-parallel: true + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + max-model-len: 9472 + max-num-seqs: 16 + enforce-eager: true + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + no-enable-chunked-prefill: true + kv-cache-dtype: fp8 + async-scheduling: true + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + decode: + nodes: 2 + workers: 2 + gpus: 4 + env: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_MOE_DP_CHUNK_SIZE: '384' + VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: '8192' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TCP_AF_PRIO: "inet" + args: + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + max-model-len: 9472 + max-num-seqs: 256 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+rms_norm"],"pass_config":{}}' + kv-cache-dtype: fp8 + all2all-backend: deepep_low_latency + async-scheduling: true + stream-interval: 32 + enable-dbo: true + dbo-decode-token-threshold: 32 + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + block-size: 128 + language-model-only: true +health_check: {max_attempts: 360, interval_seconds: 10} +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "512" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-4p2d-dep2-tp4-c1024-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-4p2d-dep2-tp4-c1024-stp.yaml new file mode 100644 index 0000000000..a799771754 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-4p2d-dep2-tp4-c1024-stp.yaml @@ -0,0 +1,129 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b200-4p2d-fp4-dep2-tp4-c1024-8k1k" +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" +resources: + gpu_type: "b200" + spread_workers: true + gpus_per_node: 8 +dynamo: {install: true, source: {pypi: 1.3.0.dev20260710}} +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: vllm + reasoning-parser: minimax_m3 +engine: + type: vllm + connector: +roles: + prefill: + nodes: 4 + workers: 4 + gpus: 2 + env: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TCP_AF_PRIO: "inet" + args: + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + enable-expert-parallel: true + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + max-model-len: 9472 + max-num-seqs: 16 + enforce-eager: true + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + no-enable-chunked-prefill: true + kv-cache-dtype: fp8 + async-scheduling: true + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + decode: + nodes: 2 + workers: 2 + gpus: 4 + env: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_MOE_DP_CHUNK_SIZE: '384' + VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: '8192' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TCP_AF_PRIO: "inet" + args: + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + max-model-len: 9472 + max-num-seqs: 512 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+rms_norm"],"pass_config":{}}' + kv-cache-dtype: fp8 + all2all-backend: deepep_low_latency + async-scheduling: true + stream-interval: 32 + enable-dbo: true + dbo-decode-token-threshold: 32 + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + block-size: 128 + language-model-only: true +health_check: {max_attempts: 360, interval_seconds: 10} +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1024" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..d596da9d23 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tep8-stp.yaml @@ -0,0 +1,85 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p1d-fp4-dep2-tep8-1k1k" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "4x16x64x128x4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tp4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tp4-stp.yaml new file mode 100644 index 0000000000..ea386d18c8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tp4-stp.yaml @@ -0,0 +1,86 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p1d-fp4-dep2-tp4-1k1k" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + UCX_TLS: "cuda_ipc,cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1x4x8x16" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p2d-dep2-dep4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p2d-dep2-dep4-stp.yaml new file mode 100644 index 0000000000..11748f090e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p2d-dep2-dep4-stp.yaml @@ -0,0 +1,87 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p2d-fp4-dep2-dep4-1k1k" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 2 + gpus: 4 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-dep8-stp.yaml new file mode 100644 index 0000000000..77ced205d1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-dep8-stp.yaml @@ -0,0 +1,87 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p1d-fp4-dep2-dep8-1k1k" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "512x4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..e19cb3c317 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-tep8-stp.yaml @@ -0,0 +1,85 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p1d-fp4-dep2-tep8-1k1k" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "32" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p2d-dep2-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p2d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..da521ba2e4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p2d-dep2-tep8-stp.yaml @@ -0,0 +1,85 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p2d-fp4-dep2-tep8-1k1k" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "16" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-3p2d-dep2-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-3p2d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..b05f3d0c94 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-3p2d-dep2-tep8-stp.yaml @@ -0,0 +1,85 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-3p2d-fp4-dep2-tep8-1k1k" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 3 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "4" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p1d-dep2-tp4-eagle3-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p1d-dep2-tp4-eagle3-mtp.yaml new file mode 100644 index 0000000000..72d890780b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p1d-dep2-tp4-eagle3-mtp.yaml @@ -0,0 +1,99 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p1d-dep2-tp4-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: fp4 + +resources: + gpu_type: b300 + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + allow_prefill_decode_colocation_across_nodes: true + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: colocate + workers: 1 + gpus: 4 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + block-size: 128 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "64" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p1d-tp1-tp4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p1d-tp1-tp4-stp.yaml new file mode 100644 index 0000000000..20ed7f618b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p1d-tp1-tp4-stp.yaml @@ -0,0 +1,78 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p1d-tp1-tp4-fp4-8k1k" +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260713 +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 1 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + enforce-eager: true + max-num-batched-tokens: 8192 + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 512 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "8x16x24x32x48x64" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p1d-tp1-tp8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p1d-tp1-tp8-stp.yaml new file mode 100644 index 0000000000..bdc516ddc0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p1d-tp1-tp8-stp.yaml @@ -0,0 +1,77 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p1d-tp1-tp8-fp4-8k1k" +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260713 +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + connector: +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 1 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + enforce-eager: true + max-num-batched-tokens: 8192 + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 8 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 256 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 256 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x2x4x8x16" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p2d-dep2-tp4-eagle3-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p2d-dep2-tp4-eagle3-mtp.yaml new file mode 100644 index 0000000000..d06f66facf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p2d-dep2-tp4-eagle3-mtp.yaml @@ -0,0 +1,99 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p2d-dep2-tp4-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: fp4 + +resources: + gpu_type: b300 + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + allow_prefill_decode_colocation_across_nodes: true + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 1 + workers: 2 + gpus: 4 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + block-size: 128 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "4x32x64" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p4d-dep2-tp4-eagle3-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p4d-dep2-tp4-eagle3-mtp.yaml new file mode 100644 index 0000000000..80dc343330 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p4d-dep2-tp4-eagle3-mtp.yaml @@ -0,0 +1,99 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p4d-dep2-tp4-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: fp4 + +resources: + gpu_type: b300 + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + allow_prefill_decode_colocation_across_nodes: true + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 4 + gpus: 4 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + block-size: 128 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "4x8x64" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p6d-dep2-tp4-eagle3-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p6d-dep2-tp4-eagle3-mtp.yaml new file mode 100644 index 0000000000..77fbebee53 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p6d-dep2-tp4-eagle3-mtp.yaml @@ -0,0 +1,99 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p6d-dep2-tp4-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: fp4 + +resources: + gpu_type: b300 + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + allow_prefill_decode_colocation_across_nodes: true + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 3 + workers: 6 + gpus: 4 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + block-size: 128 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "24x48" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-2p1d-dep2-dep4-eagle3-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-2p1d-dep2-dep4-eagle3-mtp.yaml new file mode 100644 index 0000000000..7c624d1e94 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-2p1d-dep2-dep4-eagle3-mtp.yaml @@ -0,0 +1,103 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p1d-dep2-dep4-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: fp4 + +resources: + gpu_type: b300 + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + allow_prefill_decode_colocation_across_nodes: true + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: colocate + workers: 1 + gpus: 4 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + block-size: 128 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.9 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "256x512" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-2p1d-tp1-tp4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-2p1d-tp1-tp4-stp.yaml new file mode 100644 index 0000000000..8953353b5c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-2p1d-tp1-tp4-stp.yaml @@ -0,0 +1,78 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p1d-tp1-tp4-fp4-8k1k" +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260713 +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 1 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + enforce-eager: true + max-num-batched-tokens: 8192 + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 512 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "128" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-2p3d-dep2-tp4-eagle3-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-2p3d-dep2-tp4-eagle3-mtp.yaml new file mode 100644 index 0000000000..e8dd313390 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-2p3d-dep2-tp4-eagle3-mtp.yaml @@ -0,0 +1,99 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p3d-dep2-tp4-fp4-8k1k-eagle3" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: fp4 + +resources: + gpu_type: b300 + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + allow_prefill_decode_colocation_across_nodes: true + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 1 + workers: 3 + gpus: 4 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_copy,cuda_ipc,rc + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + block-size: 128 + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-model-len: 9472 + language-model-only: true + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + req_rate: inf + num_warmup_mult: 0 + random_range_ratio: 0.8 + use_chat_template: true + concurrencies: "192" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-3p1d-tp1-tp4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-3p1d-tp1-tp4-stp.yaml new file mode 100644 index 0000000000..5b0ed3aab5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-3p1d-tp1-tp4-stp.yaml @@ -0,0 +1,78 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-3p1d-tp1-tp4-fp4-8k1k" +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260713 +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true +roles: + prefill: + nodes: 1 + workers: 3 + gpus: 1 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + enforce-eager: true + max-num-batched-tokens: 8192 + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 512 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "256" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-4p1d-tp1-tep4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-4p1d-tp1-tep4-stp.yaml new file mode 100644 index 0000000000..b8bcd74d39 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-4p1d-tp1-tep4-stp.yaml @@ -0,0 +1,78 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-4p1d-tp1-tep4-fp4-8k1k" +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260713 +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true +roles: + prefill: + nodes: 1 + workers: 4 + gpus: 1 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + enforce-eager: true + max-num-batched-tokens: 8192 + decode: + nodes: colocate + workers: 1 + gpus: 4 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-4p2d-dep2-tep4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-4p2d-dep2-tep4-stp.yaml new file mode 100644 index 0000000000..8f5b55d96b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-4p2d-dep2-tep4-stp.yaml @@ -0,0 +1,89 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-4p2d-fp4-dep2-tep4-8k1k" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" + precision: "fp4" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 4 + gpus: 2 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 1 + workers: 2 + gpus: 4 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-5p2d-tp1-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-5p2d-tp1-dep8-stp.yaml new file mode 100644 index 0000000000..f63955df72 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-5p2d-tp1-dep8-stp.yaml @@ -0,0 +1,79 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-5p2d-tp1-dep8-fp4-8k1k" +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260713 +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + connector: +roles: + prefill: + nodes: 1 + workers: 5 + gpus: 1 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + enforce-eager: true + max-num-batched-tokens: 8192 + decode: + nodes: 2 + workers: 2 + gpus: 8 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-8p2d-tp1-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-8p2d-tp1-dep8-stp.yaml new file mode 100644 index 0000000000..3027068b25 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-8p2d-tp1-dep8-stp.yaml @@ -0,0 +1,79 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-8p2d-tp1-dep8-fp4-8k1k" +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260713 +frontend: + type: dynamo + enable_multiple_frontends: false +engine: + type: vllm + connector: +roles: + prefill: + nodes: 1 + workers: 8 + gpus: 1 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + enforce-eager: true + max-num-batched-tokens: 8192 + decode: + nodes: 2 + workers: 2 + gpus: 8 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + args: + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "768x1024" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..b1bbc5f8c5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tep8-stp.yaml @@ -0,0 +1,83 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p1d-fp8-dep2-tep8-1k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "4x16x64x128x4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml new file mode 100644 index 0000000000..ec4d9fd6e2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml @@ -0,0 +1,85 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p1d-fp8-dep2-tp4-marlin-1k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + UCX_TLS: "cuda_ipc,cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: + UCX_TLS: "cuda_ipc,cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 4 + enable-expert-parallel: false + moe-backend: marlin + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1x4x8x16" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p2d-dep2-dep4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p2d-dep2-dep4-stp.yaml new file mode 100644 index 0000000000..03aa32c375 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p2d-dep2-dep4-stp.yaml @@ -0,0 +1,85 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p2d-fp8-dep2-dep4-1k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 2 + gpus: 4 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-dep8-stp.yaml new file mode 100644 index 0000000000..75325490a5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-dep8-stp.yaml @@ -0,0 +1,85 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p1d-fp8-dep2-dep8-1k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "512x4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..f2be29e938 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-tep8-stp.yaml @@ -0,0 +1,83 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p1d-fp8-dep2-tep8-1k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "32" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..67f04f0885 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml @@ -0,0 +1,83 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p2d-fp8-dep2-tep8-1k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "16" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-3p2d-dep2-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-3p2d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..a7964b126f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-3p2d-dep2-tep8-stp.yaml @@ -0,0 +1,83 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-3p2d-fp8-dep2-tep8-1k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 3 + gpus: 2 + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + UCX_TLS: "cuda_copy,rc" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + + args: + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8192 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "4" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml new file mode 100644 index 0000000000..180c581400 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml @@ -0,0 +1,89 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p1d-fp8-dep2-tp4-marlin-8k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + allow_prefill_decode_colocation: true + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_ipc,cuda_copy,rc" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_ipc,cuda_copy,rc" + + args: + tensor-parallel-size: 4 + enable-expert-parallel: false + moe-backend: marlin + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x4x8x16" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-1p2d-dep2-tep4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-1p2d-dep2-tep4-stp.yaml new file mode 100644 index 0000000000..04ef6e2919 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-1p2d-dep2-tep4-stp.yaml @@ -0,0 +1,87 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p2d-fp8-dep2-tep4-8k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 1 + workers: 2 + gpus: 4 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + + args: + tensor-parallel-size: 4 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 512 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "16" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-1p4d-dep2-tep4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-1p4d-dep2-tep4-stp.yaml new file mode 100644 index 0000000000..2a0b63c658 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-1p4d-dep2-tep4-stp.yaml @@ -0,0 +1,87 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p4d-fp8-dep2-tep4-8k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 4 + gpus: 4 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + + args: + tensor-parallel-size: 4 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 512 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "16x32x64x128" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-1p4d-dep2-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-1p4d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..b3bfa0fc68 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-1p4d-dep2-tep8-stp.yaml @@ -0,0 +1,87 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-1p4d-fp8-dep2-tep8-8k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 4 + workers: 4 + gpus: 8 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + + args: + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-2p2d-dep2-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-2p2d-dep2-dep8-stp.yaml new file mode 100644 index 0000000000..f8a78edaac --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-2p2d-dep2-dep8-stp.yaml @@ -0,0 +1,89 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p2d-fp8-dep2-dep8-8k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "256x512" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-2p2d-dep2-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-2p2d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..bf37110f5b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-2p2d-dep2-tep8-stp.yaml @@ -0,0 +1,87 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-2p2d-fp8-dep2-tep8-8k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + + args: + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "16" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-3p2d-dep2-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-3p2d-dep2-dep8-stp.yaml new file mode 100644 index 0000000000..dcd8a7575a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-3p2d-dep2-dep8-stp.yaml @@ -0,0 +1,89 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-3p2d-fp8-dep2-dep8-8k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 3 + gpus: 2 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 1024 # Per DP rank: 2 workers x DP8 = 16 ranks. + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1024x2048" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-4p2d-dep2-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-4p2d-dep2-dep8-stp.yaml new file mode 100644 index 0000000000..3546c4362a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-4p2d-dep2-dep8-stp.yaml @@ -0,0 +1,89 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-b300-4p2d-fp8-dep2-dep8-8k1k" + +model: + path: "MiniMaxAI/MiniMax-M3-MXFP8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + +resources: + gpu_type: "b300" + gpus_per_node: 8 +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 4 + gpus: 2 + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + + args: + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 2 + gpus: 8 + + env: + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: "cuda_copy,rc" + + args: + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 1024 # Per DP rank: 2 workers x DP8 = 16 ranks. + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4-vllm-simple.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4-vllm-simple.yaml new file mode 100644 index 0000000000..eb661f2c60 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4-vllm-simple.yaml @@ -0,0 +1,108 @@ +schema: 2 +name: "minimax-m3-vllm-agg-gb200-dep4-vllm-simple-agentic" + +model: {path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4"} +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: "vllm/vllm-openai:v0.27.1"} + frameworks: {dynamo: "1.3.1"} +dynamo: {install: true, source: {pypi: "1.3.1"}} +environment: {ETCD_LEASE_TTL: "7200"} +slurm: {time_limit: "12:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} +resources: {gpu_type: "gb200", gpus_per_node: 4} +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + env: {DYN_TCP_REQUEST_TIMEOUT: "60"} + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "minimax_m3" + reasoning-parser: "minimax_m3" + enable-auto-tool-choice: true + router-mode: "kv" + router-kv-events: true + router-reset-states: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + kv_events: true +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4.yaml new file mode 100644 index 0000000000..33324c4699 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep4.yaml @@ -0,0 +1,106 @@ +schema: 2 +name: "minimax-m3-vllm-agg-gb200-dep4-agentic" + +model: {path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4"} +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: "vllm/vllm-openai:v0.27.1"} + frameworks: {dynamo: "1.3.1"} +dynamo: {install: true, source: {pypi: "1.3.1"}} +environment: {ETCD_LEASE_TTL: "7200"} +slurm: {time_limit: "12:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} +resources: {gpu_type: "gb200", gpus_per_node: 4} +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + env: {DYN_TCP_REQUEST_TIMEOUT: "60"} + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "minimax_m3" + reasoning-parser: "minimax_m3" + enable-auto-tool-choice: true + router-mode: "kv" + router-kv-events: true + router-reset-states: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + kv_events: true +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep8.yaml new file mode 100644 index 0000000000..020dd3d4d0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-dep8.yaml @@ -0,0 +1,106 @@ +schema: 2 +name: "minimax-m3-vllm-agg-gb200-dep8-agentic" + +model: {path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4"} +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: "vllm/vllm-openai:v0.27.1"} + frameworks: {dynamo: "1.3.1"} +dynamo: {install: true, source: {pypi: "1.3.1"}} +environment: {ETCD_LEASE_TTL: "7200"} +slurm: {time_limit: "12:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} +resources: {gpu_type: "gb200", gpus_per_node: 4} +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + env: {DYN_TCP_REQUEST_TIMEOUT: "60"} + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "minimax_m3" + reasoning-parser: "minimax_m3" + enable-auto-tool-choice: true + router-mode: "kv" + router-kv-events: true + router-reset-states: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + kv_events: true +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml new file mode 100644 index 0000000000..5b4a692ed4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-agg-gb200-tp4-agentic-nightly-native" + +model: + path: "minimax-m3-nvfp4" + container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" + precision: "fp4" + +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7"} + frameworks: {dynamo: "1.5.0.dev20260908"} + +dynamo: {install: true, source: {pypi: "1.5.0.dev20260908"}} +environment: {ETCD_LEASE_TTL: "7200"} + +slurm: {time_limit: "12:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: {DYN_TCP_REQUEST_TIMEOUT: "60"} + args: + trust-remote-code: true + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 + +engine: + type: vllm + connector: +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + + kv_events: true +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml new file mode 100644 index 0000000000..cd56742486 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml @@ -0,0 +1,108 @@ +schema: 2 +name: "minimax-m3-vllm-agg-gb200-tp4-vllm-simple-agentic-nightly-native" + +model: + path: "minimax-m3-nvfp4" + container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" + precision: "fp4" + +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7"} + frameworks: {dynamo: "1.5.0.dev20260908"} + +dynamo: {install: true, source: {pypi: "1.5.0.dev20260908"}} +environment: {ETCD_LEASE_TTL: "7200"} +slurm: {time_limit: "12:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} +resources: {gpu_type: "gb200", gpus_per_node: 4} +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: {DYN_TCP_REQUEST_TIMEOUT: "60"} + args: + trust-remote-code: true + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 + +engine: + type: vllm + connector: +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":777389080576,"cpu_bytes_to_use_per_rank":194347270144,"lazy_offload":true}}' + stream-interval: 20 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + + kv_events: true +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple.yaml new file mode 100644 index 0000000000..f88160a9cf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple.yaml @@ -0,0 +1,113 @@ +schema: 2 +name: "minimax-m3-vllm-agg-gb200-tp4-vllm-simple-agentic" + +model: + path: "minimax-m3-nvfp4" + container: "vllm/vllm-openai:v0.27.1" + precision: "fp4" + +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: "vllm/vllm-openai:v0.27.1"} + frameworks: {dynamo: "1.3.1"} + +dynamo: {install: true, source: {pypi: "1.3.1"}} +environment: {ETCD_LEASE_TTL: "7200"} +slurm: {time_limit: "12:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} +resources: {gpu_type: "gb200", gpus_per_node: 4} +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: {DYN_TCP_REQUEST_TIMEOUT: "60"} + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "minimax_m3" + reasoning-parser: "minimax_m3" + enable-auto-tool-choice: true + router-mode: "kv" + router-kv-events: true + router-reset-states: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 + +engine: + type: vllm + connector: +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' + stream-interval: 20 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + + kv_events: true +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4.yaml new file mode 100644 index 0000000000..f8fdb3c152 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4.yaml @@ -0,0 +1,116 @@ +schema: 2 +name: "minimax-m3-vllm-agg-gb200-tp4-agentic" + +model: + path: "minimax-m3-nvfp4" + container: "vllm/vllm-openai:v0.27.1" + precision: "fp4" + +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: "vllm/vllm-openai:v0.27.1"} + frameworks: {dynamo: "1.3.1"} + +dynamo: {install: true, source: {pypi: "1.3.1"}} +environment: {ETCD_LEASE_TTL: "7200"} + +slurm: {time_limit: "12:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: {DYN_TCP_REQUEST_TIMEOUT: "60"} + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "minimax_m3" + reasoning-parser: "minimax_m3" + enable-auto-tool-choice: true + router-mode: "kv" + router-kv-events: true + router-reset-states: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 + +engine: + type: vllm + connector: +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + + kv_events: true +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml new file mode 100644 index 0000000000..590eac5eaf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-agg-gb200-tp8-agentic-nightly-native" + +model: + path: "minimax-m3-nvfp4" + container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" + precision: "fp4" + +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7"} + frameworks: {dynamo: "1.5.0.dev20260908"} + +dynamo: {install: true, source: {pypi: "1.5.0.dev20260908"}} +environment: {ETCD_LEASE_TTL: "7200"} + +slurm: {time_limit: "12:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: {DYN_TCP_REQUEST_TIMEOUT: "60"} + args: + trust-remote-code: true + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 + +engine: + type: vllm + connector: +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + stream-interval: 20 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + + kv_events: true +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep4.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep4.yaml new file mode 100644 index 0000000000..d04d2c6299 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep4.yaml @@ -0,0 +1,142 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep8-dep4-agentic" + +model: {path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4"} +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: "vllm/vllm-openai:v0.27.1"} + frameworks: {dynamo: "1.3.1"} +dynamo: {install: true, source: {pypi: "1.3.1"}} +environment: {ETCD_LEASE_TTL: "7200"} +slurm: {time_limit: "12:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + enable_multiple_frontends: false + env: {DYN_TCP_REQUEST_TIMEOUT: "60"} + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "minimax_m3" + reasoning-parser: "minimax_m3" + enable-auto-tool-choice: true + router-mode: "kv" + router-kv-events: true + router-reset-states: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: &worker_env + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: "300" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"PIECEWISE"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + kv_events: true + decode: + nodes: 1 + workers: 1 + gpus: 4 + env: *worker_env + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + kv_events: true +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml new file mode 100644 index 0000000000..20e46986bf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml @@ -0,0 +1,173 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p1d-tp4-tp4-c24-agentic" + +model: + path: "minimax-m3-nvfp4" + container: &container "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6" + precision: "fp4" + +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: *container} + frameworks: {dynamo: "1.5.0.dev20260819"} + +dynamo: {install: true, source: {wheel: "1.5.0.dev20260819"}} +environment: {PYTHONHASHSEED: "0"} +health_check: {max_attempts: 2160, interval_seconds: 10} + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "minimax_m3" + reasoning-parser: "minimax_m3" + enable-auto-tool-choice: true + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 3600 + env: + DYN_LOG: "info" + DYN_TCP_CONNECT_TIMEOUT: "120" + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE: "1" + +engine: + type: vllm + connector: + dp_launch_mode: per_gpu +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + UCX_TLS: "tcp,cuda_ipc,cuda_copy" + WITH_NVIDIA_PEERMEM: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "1" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + kv-cache-dtype: "fp8" + block-size: 128 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 512 + stream-interval: 20 + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + numa-bind: true + numa-bind-nodes: &numa_nodes [0, 0, 1, 1] + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: *worker_environment + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + kv-cache-dtype: "fp8" + block-size: 128 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 512 + stream-interval: 20 + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + numa-bind: true + numa-bind-nodes: *numa_nodes + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AGENTIC_WARMUP_GRACE_PERIOD: "1800" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "3600" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml new file mode 100644 index 0000000000..375daf4db4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml @@ -0,0 +1,180 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p1d-tp4-tp8-c1-agentic" + +model: + path: "minimax-m3-nvfp4" + container: &container "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6" + precision: "fp4" + +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: *container} + frameworks: {dynamo: "1.5.0.dev20260819"} + +dynamo: {install: true, source: {wheel: "1.5.0.dev20260819"}} +environment: {PYTHONHASHSEED: "0"} +health_check: {max_attempts: 2160, interval_seconds: 10} + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "minimax_m3" + reasoning-parser: "minimax_m3" + enable-auto-tool-choice: true + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 3600 + env: + DYN_LOG: "info" + DYN_TCP_CONNECT_TIMEOUT: "120" + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE: "1" + +engine: + type: vllm + connector: + dp_launch_mode: per_gpu +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + UCX_TLS: "tcp,cuda_ipc,cuda_copy" + WITH_NVIDIA_PEERMEM: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "1" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + kv-cache-dtype: "fp8" + block-size: 128 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 512 + stream-interval: 20 + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + numa-bind: true + numa-bind-nodes: &numa_nodes [0, 0, 1, 1] + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "1" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 8 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + kv-cache-dtype: "fp8" + block-size: 128 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-num-seqs: 1 + max-cudagraph-capture-size: 512 + stream-interval: 20 + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + numa-bind: true + numa-bind-nodes: *numa_nodes + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AGENTIC_WARMUP_GRACE_PERIOD: "1800" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "3600" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml new file mode 100644 index 0000000000..7d5ce3f56e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml @@ -0,0 +1,173 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p2d-tp4-tp4-c8-c16-agentic" + +model: + path: "minimax-m3-nvfp4" + container: &container "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6" + precision: "fp4" + +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: *container} + frameworks: {dynamo: "1.5.0.dev20260819"} + +dynamo: {install: true, source: {wheel: "1.5.0.dev20260819"}} +environment: {PYTHONHASHSEED: "0"} +health_check: {max_attempts: 2160, interval_seconds: 10} + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "minimax_m3" + reasoning-parser: "minimax_m3" + enable-auto-tool-choice: true + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 3600 + env: + DYN_LOG: "info" + DYN_TCP_CONNECT_TIMEOUT: "120" + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE: "1" + +engine: + type: vllm + connector: + dp_launch_mode: per_gpu +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + UCX_TLS: "tcp,cuda_ipc,cuda_copy" + WITH_NVIDIA_PEERMEM: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "1" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + kv-cache-dtype: "fp8" + block-size: 128 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 512 + stream-interval: 20 + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + numa-bind: true + numa-bind-nodes: &numa_nodes [0, 0, 1, 1] + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + decode: + nodes: 2 + workers: 2 + gpus: 4 + + env: *worker_environment + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + kv-cache-dtype: "fp8" + block-size: 128 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 512 + stream-interval: 20 + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + numa-bind: true + numa-bind-nodes: *numa_nodes + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AGENTIC_WARMUP_GRACE_PERIOD: "1800" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "3600" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-dep8-stp.yaml new file mode 100644 index 0000000000..2b5219894d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-dep8-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep8-fp8-1k1k" + +# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 1D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). +# GB200 has 4 GPUs/node. +# Nodes: 1 prefill + 2 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1024x4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-tep8-stp.yaml new file mode 100644 index 0000000000..7dbe4dfc34 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-tep8-stp.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-tep8-fp8-1k1k" + +# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 1D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). +# GB200 has 4 GPUs/node. +# Nodes: 1 prefill + 2 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8196 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "128x256" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-dep8-stp.yaml new file mode 100644 index 0000000000..05b31cf0cf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-dep8-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-dep8-fp8-1k1k" + +# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). +# GB200 has 4 GPUs/node. +# Nodes: 1 prefill + 4 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 4 + workers: 2 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1024x4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-tep8-stp.yaml new file mode 100644 index 0000000000..dba3eb0560 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-tep8-stp.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-1k1k" + +# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). +# GB200 has 4 GPUs/node. +# Nodes: 1 prefill + 4 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 4 + workers: 2 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8196 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "4x16x64" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p4d-dep4-tp4-marlin-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p4d-dep4-tp4-marlin-stp.yaml new file mode 100644 index 0000000000..2c272f70f3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p4d-dep4-tp4-marlin-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p4d-dep4-tp4-marlin-fp8-1k1k" + +# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 4D TP4 decode (TP4, 4 GPU/worker = 1 node each). +# Decode uses Marlin MoE backend with expert-parallel disabled (1p4d only). +# GB200 has 4 GPUs/node. +# Nodes: 1 prefill + 4 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 4 + workers: 4 + gpus: 4 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 4 + moe-backend: marlin + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8196 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "32" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-1p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-1p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..a0f9806eaf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-1p1d-dep4-dep16-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep16-fp8-8k1k" + +# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). +# GB200 has 4 GPUs/node. +# Nodes: 1 prefill + 4 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-1p2d-dep4-tep4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-1p2d-dep4-tep4-stp.yaml new file mode 100644 index 0000000000..fb2136aced --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-1p2d-dep4-tep4-stp.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep4-fp8-8k1k" + +# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D TEP4 decode (TP4 EP, 4 GPU/worker = 1 node each). +# GB200 has 4 GPUs/node. +# Nodes: 1 prefill + 2 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 2 + gpus: 4 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 4 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-num-seqs: 512 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-1p2d-dep4-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-1p2d-dep4-tep8-stp.yaml new file mode 100644 index 0000000000..283a1b30de --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-1p2d-dep4-tep8-stp.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-8k1k" + +# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). +# GB200 has 4 GPUs/node. +# Nodes: 1 prefill + 4 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 4 + workers: 2 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4x16x32x64x128" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-2p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-2p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..3e32e00e91 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-2p1d-dep4-dep16-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep16-fp8-8k1k" + +# 2P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). +# GB200 has 4 GPUs/node. +# Nodes: 1 prefill + 4 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512x1024" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-3p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-3p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..9d1a932828 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-3p1d-dep4-dep16-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-3p1d-dep4-dep16-fp8-8k1k" + +# 3P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). +# GB200 has 4 GPUs/node. +# Nodes: 2 prefill + 4 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 3 + workers: 3 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-5p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-5p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..88cf27be55 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-5p1d-dep4-dep16-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-5p1d-dep4-dep16-fp8-8k1k" + +# 5P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). +# GB200 has 4 GPUs/node. +# Nodes: 3 prefill + 4 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 5 + workers: 5 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml new file mode 100644 index 0000000000..44e9395766 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml @@ -0,0 +1,179 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-eval-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + install: true + + source: + pypi: "1.4.0.dev20260730" +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: false + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml new file mode 100644 index 0000000000..2d2ab264ee --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml @@ -0,0 +1,179 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + install: true + + source: + pypi: "1.4.0.dev20260730" +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: false + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml new file mode 100644 index 0000000000..88bc24e782 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml @@ -0,0 +1,179 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p1d-tp2-tp4-c20-c24-fp4-eval-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + install: true + + source: + pypi: "1.4.0.dev20260730" +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml new file mode 100644 index 0000000000..3ec7248ec7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml @@ -0,0 +1,179 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p1d-tp2-tp4-c20-c24-fp4-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + install: true + + source: + pypi: "1.4.0.dev20260730" +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml new file mode 100644 index 0000000000..1e6fadc381 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml @@ -0,0 +1,182 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p3d-dep4-tp4-c24-fp4-eval-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + install: true + + source: + pypi: "1.4.0.dev20260730" +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + nodes: 3 + workers: 3 + gpus: 4 + + env: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml new file mode 100644 index 0000000000..f35f8efb3c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml @@ -0,0 +1,182 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p3d-dep4-tp4-c24-fp4-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + install: true + + source: + pypi: "1.4.0.dev20260730" +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + nodes: 3 + workers: 3 + gpus: 4 + + env: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 4 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml new file mode 100644 index 0000000000..d0e7aa0acf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml @@ -0,0 +1,179 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p3d-tp2-tp2-c48-fp4-eval-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + install: true + + source: + pypi: "1.4.0.dev20260730" +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: true +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + nodes: 3 + workers: 3 + gpus: 2 + + env: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml new file mode 100644 index 0000000000..68c4446a9c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml @@ -0,0 +1,179 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p3d-tp2-tp2-c48-fp4-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + install: true + + source: + pypi: "1.4.0.dev20260730" +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: true +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + nodes: 3 + workers: 3 + gpus: 2 + + env: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml new file mode 100644 index 0000000000..2ca720c02e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml @@ -0,0 +1,179 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-2p5d-tp2-tp2-c120-fp4-eval-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + install: true + + source: + pypi: "1.4.0.dev20260730" +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: true +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 2 + env: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + nodes: 5 + workers: 5 + gpus: 2 + + env: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml new file mode 100644 index 0000000000..1f0f2657d2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml @@ -0,0 +1,179 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-2p5d-tp2-tp2-c120-fp4-agentic" + +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: + repo: "nvidia/MiniMax-M3-NVFP4" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + +dynamo: + install: true + + source: + pypi: "1.4.0.dev20260730" +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: true +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "200GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + env: + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_TCP_CONNECT_TIMEOUT: "120" + +engine: + type: vllm + connector: + dp_launch_mode: per_node +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 2 + env: &worker_environment + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + DYN_TCP_CONNECT_TIMEOUT: "120" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + UCX_MEMTYPE_CACHE: "n" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' + enable-cumem-allocator: true + decode: + nodes: 5 + workers: 5 + gpus: 2 + + env: + <<: *worker_environment + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + kv-cache-dtype: "fp8" + block-size: 128 + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 1024 + stream-interval: 20 + gpu-memory-utilization: 0.9 + tensor-parallel-size: 2 + enable-expert-parallel: false + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + enable-cumem-allocator: true +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p1d-dep2-dep4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p1d-dep2-dep4-stp.yaml new file mode 100644 index 0000000000..c5a54c7001 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p1d-dep2-dep4-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-dep4-fp8-1k1k" + +# 1P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D DEP4 decode (TP1 DP4 EP, 4 GPU/worker = 1 node each). +# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. +# Nodes: 1 prefill + 1 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "8192" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p2d-dep2-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p2d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..f730553b39 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p2d-dep2-tep8-stp.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p2d-dep2-tep8-fp8-1k1k" + +# 1P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). +# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. +# Nodes: 1 prefill + 4 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 4 + workers: 2 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8196 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "4x16x64x128x256" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..1d82dbc63a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-2p2d-dep2-tep8-fp8-1k1k" + +# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). +# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. +# Nodes: 1 prefill + 4 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 4 + workers: 2 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 8196 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "32" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p3d-dep2-dep4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p3d-dep2-dep4-stp.yaml new file mode 100644 index 0000000000..e35c79b12b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p3d-dep2-dep4-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-2p3d-dep2-dep4-fp8-1k1k" + +# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 3D DEP4 decode (TP1 DP4 EP, 4 GPU/worker = 1 node each). +# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. +# Nodes: 1 prefill + 3 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 3 + workers: 3 + gpus: 4 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "8192" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p4d-dep2-dep4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p4d-dep2-dep4-stp.yaml new file mode 100644 index 0000000000..80dd0dd062 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p4d-dep2-dep4-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-2p4d-dep2-dep4-fp8-1k1k" + +# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 4D DEP4 decode (TP1 DP4 EP, 4 GPU/worker = 1 node each). +# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. +# Nodes: 1 prefill + 4 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 4 + workers: 4 + gpus: 4 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "8192" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-4p2d-dep2-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-4p2d-dep2-dep8-stp.yaml new file mode 100644 index 0000000000..f2ca301f29 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-4p2d-dep2-dep8-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-4p2d-dep2-dep8-fp8-1k1k" + +# 4P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). +# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. +# Nodes: 2 prefill + 4 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260614 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 2 + workers: 4 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 2048 + + decode: + nodes: 4 + workers: 2 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 2304 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 4096 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1024x4096" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-dep8-eagle3-c64-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-dep8-eagle3-c64-mtp.yaml new file mode 100644 index 0000000000..874d7acce7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-dep8-eagle3-c64-mtp.yaml @@ -0,0 +1,120 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-dep8-mxfp8-8k1k-eagle3-c64" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + source: + pypi: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: "dynamo" + enable_multiple_frontends: false + +engine: + type: "vllm" + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: *worker-environment + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "64" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-dep8-stp.yaml new file mode 100644 index 0000000000..cf55baea4f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-dep8-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-dep8-fp8-8k1k" + +# 1P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). +# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. +# Nodes: 1 prefill + 2 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "256" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..220841cfc0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tep8-stp.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tep8-fp8-8k1k" + +# 1P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). +# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. +# Nodes: 1 prefill + 2 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "128" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp4-eagle3-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp4-eagle3-c1-mtp.yaml new file mode 100644 index 0000000000..461663dbcf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp4-eagle3-c1-mtp.yaml @@ -0,0 +1,116 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp4-mxfp8-8k1k-eagle3-c1" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + source: + pypi: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: "dynamo" + enable_multiple_frontends: false + +engine: + type: "vllm" + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: *worker-environment + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 4 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp4-eagle3-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp4-eagle3-c8-mtp.yaml new file mode 100644 index 0000000000..6a23ea1c6e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp4-eagle3-c8-mtp.yaml @@ -0,0 +1,116 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp4-mxfp8-8k1k-eagle3-c8" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + source: + pypi: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: "dynamo" + enable_multiple_frontends: false + +engine: + type: "vllm" + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 1 + workers: 1 + gpus: 4 + + env: *worker-environment + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 4 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "8" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp8-eagle3-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp8-eagle3-c1-mtp.yaml new file mode 100644 index 0000000000..99c39d0523 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp8-eagle3-c1-mtp.yaml @@ -0,0 +1,116 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp8-mxfp8-8k1k-eagle3-c1" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + source: + pypi: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: "dynamo" + enable_multiple_frontends: false + +engine: + type: "vllm" + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: *worker-environment + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 8 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp8-eagle3-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp8-eagle3-c4-mtp.yaml new file mode 100644 index 0000000000..f6b0a7f7d4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp8-eagle3-c4-mtp.yaml @@ -0,0 +1,116 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp8-mxfp8-8k1k-eagle3-c4" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + source: + pypi: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: "dynamo" + enable_multiple_frontends: false + +engine: + type: "vllm" + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: *worker-environment + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 8 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp8-eagle3-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp8-eagle3-c8-mtp.yaml new file mode 100644 index 0000000000..af228ee966 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp8-eagle3-c8-mtp.yaml @@ -0,0 +1,116 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp8-mxfp8-8k1k-eagle3-c8" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + source: + pypi: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: "dynamo" + enable_multiple_frontends: false + +engine: + type: "vllm" + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: *worker-environment + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 8 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "8" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p2d-dep2-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p2d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..05d24c1e03 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p2d-dep2-tep8-stp.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-1p2d-dep2-tep8-fp8-8k1k" + +# 1P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). +# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. +# Nodes: 1 prefill + 4 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 4 + workers: 2 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "32x64x128" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-2p1d-dep2-dep8-eagle3-c512-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-2p1d-dep2-dep8-eagle3-c512-mtp.yaml new file mode 100644 index 0000000000..6d4c2d36b5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-2p1d-dep2-dep8-eagle3-c512-mtp.yaml @@ -0,0 +1,120 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-2p1d-dep2-dep8-mxfp8-8k1k-eagle3-c512" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + source: + pypi: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: "dynamo" + enable_multiple_frontends: false + +engine: + type: "vllm" + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: *worker-environment + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-2p1d-dep2-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-2p1d-dep2-dep8-stp.yaml new file mode 100644 index 0000000000..a5bc97e8f9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-2p1d-dep2-dep8-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-2p1d-dep2-dep8-fp8-8k1k" + +# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). +# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. +# Nodes: 1 prefill + 2 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-2p2d-dep2-tep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-2p2d-dep2-tep8-stp.yaml new file mode 100644 index 0000000000..16d4223a1c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-2p2d-dep2-tep8-stp.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-2p2d-dep2-tep8-fp8-8k1k" + +# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). +# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. +# Nodes: 1 prefill + 4 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 4 + workers: 2 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 8 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "16" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-2p4d-dep2-tep4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-2p4d-dep2-tep4-stp.yaml new file mode 100644 index 0000000000..1b4b2d4f40 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-2p4d-dep2-tep4-stp.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-2p4d-dep2-tep4-fp8-8k1k" + +# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 4D TEP4 decode (TP4 EP, 4 GPU/worker = 1 node each). +# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. +# Nodes: 1 prefill + 4 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 4 + workers: 4 + gpus: 4 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 4 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 32 + max-num-seqs: 512 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 4096 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-3p1d-dep2-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-3p1d-dep2-dep16-stp.yaml new file mode 100644 index 0000000000..6c38439a44 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-3p1d-dep2-dep16-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-3p1d-dep2-dep16-fp8-8k1k" + +# 3P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). +# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. +# Nodes: 2 prefill + 4 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 2 + workers: 3 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 4 + workers: 1 + gpus: 16 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-3p1d-dep2-dep8-eagle3-c256-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-3p1d-dep2-dep8-eagle3-c256-mtp.yaml new file mode 100644 index 0000000000..0799da6ea2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-3p1d-dep2-dep8-eagle3-c256-mtp.yaml @@ -0,0 +1,120 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-3p1d-dep2-dep8-mxfp8-8k1k-eagle3-c256" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + source: + pypi: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: "dynamo" + enable_multiple_frontends: false + +engine: + type: "vllm" + connector: + +roles: + prefill: + nodes: 2 + workers: 3 + gpus: 2 + env: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: *worker-environment + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "256" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-3p1d-dep2-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-3p1d-dep2-dep8-stp.yaml new file mode 100644 index 0000000000..3857bc9952 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-3p1d-dep2-dep8-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-3p1d-dep2-dep8-fp8-8k1k" + +# 3P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). +# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. +# Nodes: 2 prefill + 2 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 2 + workers: 3 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1024" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-4p1d-dep2-dep8-eagle3-c1024-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-4p1d-dep2-dep8-eagle3-c1024-mtp.yaml new file mode 100644 index 0000000000..7db41dbe7d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-4p1d-dep2-dep8-eagle3-c1024-mtp.yaml @@ -0,0 +1,120 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-4p1d-dep2-dep8-mxfp8-8k1k-eagle3-c1024" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + source: + pypi: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: "dynamo" + enable_multiple_frontends: false + +engine: + type: "vllm" + connector: + +roles: + prefill: + nodes: 2 + workers: 4 + gpus: 2 + env: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: *worker-environment + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1024" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-4p1d-dep2-dep8-eagle3-c2048-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-4p1d-dep2-dep8-eagle3-c2048-mtp.yaml new file mode 100644 index 0000000000..1bcfa20376 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-4p1d-dep2-dep8-eagle3-c2048-mtp.yaml @@ -0,0 +1,120 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-4p1d-dep2-dep8-mxfp8-8k1k-eagle3-c2048" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + source: + pypi: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: "dynamo" + enable_multiple_frontends: false + +engine: + type: "vllm" + connector: + +roles: + prefill: + nodes: 2 + workers: 4 + gpus: 2 + env: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: *worker-environment + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-6p1d-dep2-dep8-eagle3-c2048-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-6p1d-dep2-dep8-eagle3-c2048-mtp.yaml new file mode 100644 index 0000000000..4c271e77fa --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-6p1d-dep2-dep8-eagle3-c2048-mtp.yaml @@ -0,0 +1,120 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-6p1d-dep2-dep8-mxfp8-8k1k-eagle3-c2048" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + source: + pypi: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: "dynamo" + enable_multiple_frontends: false + +engine: + type: "vllm" + connector: + +roles: + prefill: + nodes: 3 + workers: 6 + gpus: 2 + env: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: *worker-environment + args: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": + "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-6p1d-dep2-dep8-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-6p1d-dep2-dep8-stp.yaml new file mode 100644 index 0000000000..80f539e419 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-6p1d-dep2-dep8-stp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb300-6p1d-dep2-dep8-fp8-8k1k" + +# 6P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). +# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. +# Nodes: 3 prefill + 2 decode (+ head/infra). + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" + precision: "fp8" + + +dynamo: + install: true + source: + pypi: 1.3.0.dev20260710 + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +frontend: + type: dynamo + enable_multiple_frontends: false + +engine: + type: vllm + connector: + +roles: + prefill: + nodes: 3 + workers: 6 + gpus: 2 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + nodes: 2 + workers: 1 + gpus: 8 + + env: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + args: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: fp8 + stream-interval: 100 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml new file mode 100644 index 0000000000..6912536759 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml @@ -0,0 +1,115 @@ +schema: 2 +name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-cap48 + +model: {path: qwen3.5-fp4, container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b, precision: fp4} +identity: + model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} + container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} +slurm: {time_limit: "8:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} +resources: {gpu_type: gb200, gpus_per_node: 4} +services: + - name: nats + type: nats + options: + max_payload_mb: 8 +dynamo: + install: true + source: + wheel: 1.4.2 + request_plane: tcp +environment: + PIP_BREAK_SYSTEM_PACKAGES: "1" +frontend: + type: dynamo + enable_multiple_frontends: false + env: {} + args: + router-mode: round-robin + router-session-affinity-ttl-secs: 3600 + +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + PYTHONNOUSERSITE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_ENABLE_SPEC_V2: "1" + SGL_ENABLE_JIT_DEEPGEMM: "false" + TORCH_CUDA_ARCH_LIST: "10.0" + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + enable-symm-mem: false + quantization: modelopt_fp4 + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-radix-cache-strategy: extra_buffer_lazy + mamba-track-interval: 1048576 + attention-backend: trtllm_mha + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + moe-runner-backend: flashinfer_trtllm + speculative-moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 8 + cuda-graph-max-bs: 48 + max-running-requests: 48 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.88 + max-mamba-cache-size: 192 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 2.5 + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-write-policy: write_through_selective + mamba-max-states-per-path: 1 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + weight-loader-drop-cache-after-load: false + model-loader-extra-config: '{"enable_multithread_load":true}' + watchdog-timeout: 1000000 + enable-metrics: true + enable-cache-report: true + +sbatch_directives: {mem: "0", cpus-per-task: "144"} +srun_options: {mem: "0", container-remap-root: ""} +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "2" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml new file mode 100644 index 0000000000..624e4e7f68 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml @@ -0,0 +1,116 @@ +schema: 2 +name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-k3-baseline + +model: {path: qwen3.5-fp4, container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b, precision: fp4} +identity: + model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} + container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} +slurm: {time_limit: "8:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} +resources: {gpu_type: gb200, gpus_per_node: 4} +services: + - name: nats + type: nats + options: + max_payload_mb: 8 +dynamo: + install: true + source: + wheel: 1.4.2 + request_plane: tcp +environment: + PIP_BREAK_SYSTEM_PACKAGES: "1" +frontend: + type: dynamo + enable_multiple_frontends: false + env: {} + args: + router-mode: round-robin + router-session-affinity-ttl-secs: 3600 + +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + PYTHONNOUSERSITE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_ENABLE_SPEC_V2: "1" + SGL_ENABLE_JIT_DEEPGEMM: "false" + TORCH_CUDA_ARCH_LIST: "10.0" + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + enable-symm-mem: false + quantization: modelopt_fp4 + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-radix-cache-strategy: extra_buffer_lazy + mamba-track-interval: 1048576 + attention-backend: trtllm_mha + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + moe-runner-backend: flashinfer_trtllm + speculative-moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 8 + cuda-graph-max-bs: 64 + max-running-requests: 80 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.88 + max-mamba-cache-size: 320 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 1 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 2.5 + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-write-policy: write_through_selective + mamba-max-states-per-path: 1 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + weight-loader-drop-cache-after-load: false + model-loader-extra-config: '{"enable_multithread_load":true}' + watchdog-timeout: 1000000 + enable-metrics: true + enable-cache-report: true + +sbatch_directives: {mem: "0", cpus-per-task: "144"} +srun_options: {mem: "0", container-remap-root: ""} +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "2" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml new file mode 100644 index 0000000000..35991d1a61 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml @@ -0,0 +1,117 @@ +schema: 2 +name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-k5 + +model: {path: qwen3.5-fp4, container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b, precision: fp4} +identity: + model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} + container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} +slurm: {time_limit: "8:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} +resources: {gpu_type: gb200, gpus_per_node: 4} +services: + - name: nats + type: nats + options: + max_payload_mb: 8 +dynamo: + install: true + source: + wheel: 1.4.2 + request_plane: tcp +environment: + PIP_BREAK_SYSTEM_PACKAGES: "1" +frontend: + type: dynamo + enable_multiple_frontends: false + env: {} + args: + router-mode: round-robin + router-session-affinity-ttl-secs: 3600 + +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + PYTHONNOUSERSITE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_ENABLE_SPEC_V2: "1" + SGL_ENABLE_JIT_DEEPGEMM: "false" + TORCH_CUDA_ARCH_LIST: "10.0" + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + enable-symm-mem: false + quantization: modelopt_fp4 + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-radix-cache-strategy: extra_buffer_lazy + mamba-track-interval: 1048576 + attention-backend: trtllm_mha + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + moe-runner-backend: flashinfer_trtllm + speculative-moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 8 + cuda-graph-max-bs: 64 + max-running-requests: 80 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.88 + # extra_buffer_lazy uses four physical Mamba state slots per running request. + # This layout is the measured K5 middle frontier through C28. + max-mamba-cache-size: 320 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 2.5 + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-write-policy: write_through_selective + mamba-max-states-per-path: 1 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + weight-loader-drop-cache-after-load: false + model-loader-extra-config: '{"enable_multithread_load":true}' + watchdog-timeout: 1000000 + enable-metrics: true + enable-cache-report: true + +sbatch_directives: {mem: "0", cpus-per-task: "144"} +srun_options: {mem: "0", container-remap-root: ""} +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "2" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-hicache.yaml new file mode 100644 index 0000000000..eab7f2107d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-hicache.yaml @@ -0,0 +1,88 @@ +schema: 2 +name: qwen35-gb200-sglang-agentic-mtp-agg-tp4-hicache + +model: {path: qwen3.5-fp4, container: dynamo-sglang, precision: fp4} +identity: + model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} + container: {image: lmsysorg/sglang:v0.5.17-cu130} +slurm: {time_limit: "8:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} +resources: {gpu_type: gb200, gpus_per_node: 4} +frontend: + type: sglang-router + args: + worker-startup-timeout-secs: 3600 + +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + PYTHONNOUSERSITE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_ENABLE_SPEC_V2: "1" + SGL_ENABLE_JIT_DEEPGEMM: "false" + TORCH_CUDA_ARCH_LIST: "10.0" + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + linear-attn-decode-backend: flashinfer + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 64 + max-running-requests: 160 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.78 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.70 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + enable-metrics: true + enable-cache-report: true + +sbatch_directives: {mem: "0", cpus-per-task: "144"} +srun_options: {mem: "0", container-remap-root: ""} +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml new file mode 100644 index 0000000000..db3e9116f2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5-no-symm + +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b + precision: fp4 + +identity: + model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} + container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} + +slurm: {time_limit: "8:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} + +resources: + gpu_type: gb200 + gpus_per_node: 4 +services: + - name: nats + type: nats + options: + max_payload_mb: 8 +dynamo: + install: true + source: + wheel: 1.4.2 + request_plane: tcp +environment: + PIP_BREAK_SYSTEM_PACKAGES: "1" +frontend: + type: dynamo + enable_multiple_frontends: false + env: {} + args: + router-mode: round-robin + router-session-affinity-ttl-secs: 3600 + +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + + env: + PYTHONNOUSERSITE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_ENABLE_SPEC_V2: "1" + SGL_ENABLE_JIT_DEEPGEMM: "false" + TORCH_CUDA_ARCH_LIST: "10.0" + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: false + quantization: modelopt_fp4 + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + linear-attn-decode-backend: flashinfer + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + cuda-graph-max-bs: 64 + max-running-requests: 80 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.80 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + mamba-max-states-per-path: 1 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + weight-loader-drop-cache-after-load: false + model-loader-extra-config: '{"enable_multithread_load":true}' + enable-metrics: true + enable-cache-report: true + +sbatch_directives: {mem: "0", cpus-per-task: "144"} +srun_options: {mem: "0", container-remap-root: ""} + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml new file mode 100644 index 0000000000..4697c70678 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml @@ -0,0 +1,114 @@ +schema: 2 +name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5-parity + +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b + precision: fp4 + +identity: + model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} + container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} + +slurm: {time_limit: "8:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} + +resources: + gpu_type: gb200 + gpus_per_node: 4 +services: + - name: nats + type: nats + options: + max_payload_mb: 8 +dynamo: + install: true + source: + wheel: 1.4.2 + request_plane: tcp +environment: + PIP_BREAK_SYSTEM_PACKAGES: "1" +frontend: + type: dynamo + enable_multiple_frontends: false + env: {} + args: + router-mode: round-robin + router-session-affinity-ttl-secs: 3600 + +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + + env: + PYTHONNOUSERSITE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_ENABLE_SPEC_V2: "1" + SGL_ENABLE_JIT_DEEPGEMM: "false" + TORCH_CUDA_ARCH_LIST: "10.0" + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer_lazy + mamba-track-interval: 1048576 + attention-backend: trtllm_mha + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 8 + cuda-graph-max-bs: 64 + max-running-requests: 80 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.80 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + mamba-max-states-per-path: 1 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + weight-loader-drop-cache-after-load: false + model-loader-extra-config: '{"enable_multithread_load":true}' + enable-metrics: true + enable-cache-report: true + +sbatch_directives: {mem: "0", cpus-per-task: "144"} +srun_options: {mem: "0", container-remap-root: ""} + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml new file mode 100644 index 0000000000..ea6ea024e3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml @@ -0,0 +1,111 @@ +schema: 2 +name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5 + +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b + precision: fp4 + +identity: + model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2} + container: {image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b} + +slurm: {time_limit: "8:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} + +resources: + gpu_type: gb200 + gpus_per_node: 4 +services: + - name: nats + type: nats + options: + max_payload_mb: 8 +dynamo: + install: true + source: + wheel: 1.4.2 + request_plane: tcp +environment: + PIP_BREAK_SYSTEM_PACKAGES: "1" +frontend: + type: dynamo + enable_multiple_frontends: false + env: {} + args: + router-mode: round-robin + router-session-affinity-ttl-secs: 3600 + +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + + env: + PYTHONNOUSERSITE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_ENABLE_SPEC_V2: "1" + SGL_ENABLE_JIT_DEEPGEMM: "false" + TORCH_CUDA_ARCH_LIST: "10.0" + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + fp4-gemm-backend: flashinfer_cutlass + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + linear-attn-decode-backend: flashinfer + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + cuda-graph-max-bs: 64 + max-running-requests: 80 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.80 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + mamba-max-states-per-path: 1 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + weight-loader-drop-cache-after-load: false + model-loader-extra-config: '{"enable_multithread_load":true}' + enable-metrics: true + enable-cache-report: true + +sbatch_directives: {mem: "0", cpus-per-task: "144"} +srun_options: {mem: "0", container-remap-root: ""} + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..33a33c42f2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml @@ -0,0 +1,168 @@ +schema: 2 +name: "qwen3.5-1p1d-dep4-dep16" + +setup_script: rebuild-deepep.sh + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 4096 + mem-fraction-static: 0.80 + max-mamba-cache-size: 2048 + max-running-requests: 2048 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "512x1024x2048" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml new file mode 100644 index 0000000000..3f18d9b038 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml @@ -0,0 +1,136 @@ +schema: 2 +name: "qwen3.5-1p1d-tp4-tp4" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 4096 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "prefill" + + decode: + nodes: 1 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_HEALTH_CHECK_TIMEOUT: "3600" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 4096 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "decode" + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 1 + random_range_ratio: 0.8 + concurrencies: "1x2x4x8x16x32x64" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..42f45fbe75 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml @@ -0,0 +1,168 @@ +schema: 2 +name: "qwen3.5-2p1d-dep4-dep16" + +setup_script: rebuild-deepep.sh + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 2 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 2 + workers: 2 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 8192 + mem-fraction-static: 0.75 + max-mamba-cache-size: 4096 + max-running-requests: 4096 + cuda-graph-max-bs: 256 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "4096" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x2x8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x2x8-mtp.yaml new file mode 100644 index 0000000000..15a879aae0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x2x8-mtp.yaml @@ -0,0 +1,156 @@ +# Qwen3.5 FP8 GB200 disaggregated MTP 1P1D TP4/TP4 topology. + +schema: 2 +name: "qwen3.5-fp8-gb200-mtp-8k1k-1p1d-tp4-tp4" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 16384 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "prefill" + + decode: + nodes: 1 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + max-mamba-cache-size: 256 + moe-runner-backend: "flashinfer_trtllm" + + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + disable-radix-cache: true + max-running-requests: 128 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 16384 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "decode" + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "1x2x8" + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml new file mode 100644 index 0000000000..76ca5a660f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml @@ -0,0 +1,170 @@ +# Qwen3.5 FP8 GB200 disaggregated MTP 1P1D TEP8/TEP8 points. + +schema: 2 +name: "qwen3.5-fp8-gb200-mtp-8k1k-1p1d-tep8-tep8" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + max-total-tokens: 128000 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 9236 + cuda-graph-max-bs: 320 + scheduler-recv-interval: 10 + decode-log-interval: 50 + stream-interval: 50 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "mooncake" + + decode: + nodes: 2 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + max-mamba-cache-size: 1024 + moe-runner-backend: "flashinfer_trtllm" + + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + max-total-tokens: 2200000 + chunked-prefill-size: 4096 + max-prefill-tokens: 16384 + context-length: 9236 + cuda-graph-max-bs: 320 + scheduler-recv-interval: 10 + decode-log-interval: 50 + stream-interval: 50 + disaggregation-mode: "decode" + disaggregation-transfer-backend: "mooncake" + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 20 + num_warmup_mult: 2 + random_range_ratio: 0.8 + concurrencies: "32x48x80" + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml new file mode 100644 index 0000000000..d879bc8e45 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml @@ -0,0 +1,134 @@ +schema: 2 +name: "qwen3.5-1p1d-tp4-tp4" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 16384 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "prefill" + + decode: + nodes: 1 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_HEALTH_CHECK_TIMEOUT: "3600" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 16384 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "decode" + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "1x2x4x8x16x32x64x128" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml new file mode 100644 index 0000000000..73f07cd005 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml @@ -0,0 +1,186 @@ +# Qwen3.5 FP8 GB200 disaggregated MTP 3P1D DEP4/DEP16 point. + +schema: 2 +name: "qwen3.5-fp8-gb200-mtp-8k1k-3p1d-dep4-dep16" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 2 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 3 + workers: 3 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.7 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 16384 + mem-fraction-static: 0.7 + max-mamba-cache-size: 1024 + max-running-requests: 1024 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 20 + num_warmup_mult: 2 + random_range_ratio: 0.8 + concurrencies: "480" + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..e2c73265a4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml @@ -0,0 +1,173 @@ +schema: 2 +name: "qwen3.5-4p1d-dep4-dep16" + +setup_script: rebuild-deepep.sh + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 3 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 4 + workers: 4 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 16384 + mem-fraction-static: 0.80 + max-mamba-cache-size: 2048 + max-running-requests: 2048 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 20 + num_warmup_mult: 2 + random_range_ratio: 0.8 + concurrencies: "1024" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml new file mode 100644 index 0000000000..15d0c1292a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml @@ -0,0 +1,186 @@ +# Qwen3.5 FP8 GB200 disaggregated MTP 4P1D DEP4/DEP16 point. + +schema: 2 +name: "qwen3.5-fp8-gb200-mtp-8k1k-4p1d-dep4-dep16" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 3 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 4 + workers: 4 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.7 + chunked-prefill-size: 98304 + max-prefill-tokens: 24576 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 16384 + mem-fraction-static: 0.7 + max-mamba-cache-size: 1024 + max-running-requests: 1024 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 20 + num_warmup_mult: 2 + random_range_ratio: 0.8 + concurrencies: "768" + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml new file mode 100644 index 0000000000..2f359c52d8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml @@ -0,0 +1,186 @@ +# Qwen3.5 FP8 GB200 disaggregated MTP 6P1D DEP4/DEP16 point. + +schema: 2 +name: "qwen3.5-fp8-gb200-mtp-8k1k-6p1d-dep4-dep16" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 5 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 6 + workers: 6 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31001 + mem-fraction-static: 0.7 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31001 + + chunked-prefill-size: 4096 + context-length: 16384 + mem-fraction-static: 0.7 + max-mamba-cache-size: 1024 + max-running-requests: 1024 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 20 + num_warmup_mult: 2 + random_range_ratio: 0.8 + concurrencies: "1280" + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml new file mode 100644 index 0000000000..d06f231244 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml @@ -0,0 +1,186 @@ +# Qwen3.5 FP8 GB200 disaggregated MTP 7P1D DEP4/DEP16 point. + +schema: 2 +name: "qwen3.5-fp8-gb200-mtp-8k1k-7p1d-dep4-dep16" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 6 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 7 + workers: 7 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31001 + mem-fraction-static: 0.7 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31001 + + chunked-prefill-size: 4096 + context-length: 16384 + mem-fraction-static: 0.7 + max-mamba-cache-size: 2048 + max-running-requests: 2048 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 20 + num_warmup_mult: 2 + random_range_ratio: 0.8 + concurrencies: "1344" + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-8p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-8p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..eecd56d73c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-8p1d-dep4-dep16-stp.yaml @@ -0,0 +1,169 @@ +schema: 2 +name: "qwen3.5-8p1d-dep4-dep16" + +setup_script: rebuild-deepep.sh + +sbatch_directives: + mem: "0" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 8 + workers: 8 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 16384 + mem-fraction-static: 0.80 + max-mamba-cache-size: 2048 + max-running-requests: 2048 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "2048x4096" diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml new file mode 100644 index 0000000000..3e54b0d734 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml @@ -0,0 +1,186 @@ +# Qwen3.5 FP8 GB200 disaggregated MTP 8P1D DEP4/DEP16 points. + +schema: 2 +name: "qwen3.5-fp8-gb200-mtp-8k1k-8p1d-dep4-dep16" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 7 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp8" + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 8 + workers: 8 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31001 + mem-fraction-static: 0.7 + chunked-prefill-size: 98304 + max-prefill-tokens: 24576 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31001 + + chunked-prefill-size: 4096 + context-length: 16384 + mem-fraction-static: 0.7 + max-mamba-cache-size: 2048 + max-running-requests: 2048 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 20 + num_warmup_mult: 2 + random_range_ratio: 0.8 + concurrencies: "1920x2304" + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x4x8x16x32x64x256-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x4x8x16x32x64x256-stp.yaml new file mode 100644 index 0000000000..9933e469d4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x4x8x16x32x64x256-stp.yaml @@ -0,0 +1,190 @@ +# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 1P1D: TP4 Prefill + TP4 Decode +# Pure tensor parallel, no expert parallel (STP) +# 8k1k sa-bench concurrency sweep on GB300 +# +# Values taken from ni_experiment_config of the +# sa-qwen-3.5-8k1k-fp4-baseline-low-latency study, row +# qwen3.5-1p_tp4x1d_tp4-aligned-ccsweep (CSV pareto export 2026-06-05). + +schema: 2 +name: "gb300-fp4-qwen3.5_8k1k_lowlat_0" + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + source: + pypi: "1.1.0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 2 + nginx_container: nginx + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_ENABLE_SPEC_V2: "1" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_ENABLE_JIT_DEEPGEMM: "true" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + FLASHINFER_DISABLE_VERSION_CHECK: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + model-path: "/model/" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + reasoning-parser: "qwen3" + tool-call-parser: "qwen3_coder" + + quantization: "modelopt_fp4" + fp4-gemm-backend: "flashinfer_cutlass" + kv-cache-dtype: "fp8_e4m3" + + mamba-scheduler-strategy: "no_buffer" + mamba-ssm-dtype: "bfloat16" + mamba-track-interval: 2048 + + attention-backend: "trtllm_mha" + mm-attention-backend: "triton_attn" + moe-runner-backend: "flashinfer_trtllm" + linear-attn-decode-backend: "flashinfer" + + disaggregation-mode: "prefill" + disable-radix-cache: true + + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 128000 + max-running-requests: 128 + cuda-graph-max-bs: 4 + chunked-prefill-size: 32768 + max-prefill-tokens: 32768 + scheduler-recv-interval: 10 + stream-interval: 30 + load-balance-method: "round_robin" + page-size: 64 + watchdog-timeout: 1000000 + log-level: "info" + + decode: + nodes: 1 + workers: 1 + + env: + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_ENABLE_SPEC_V2: "1" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_ENABLE_JIT_DEEPGEMM: "true" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + FLASHINFER_DISABLE_VERSION_CHECK: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + model-path: "/model/" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + reasoning-parser: "qwen3" + tool-call-parser: "qwen3_coder" + + quantization: "modelopt_fp4" + fp4-gemm-backend: "flashinfer_cutlass" + kv-cache-dtype: "fp8_e4m3" + + mamba-scheduler-strategy: "no_buffer" + mamba-ssm-dtype: "bfloat16" + mamba-track-interval: 128 + + attention-backend: "trtllm_mha" + mm-attention-backend: "triton_attn" + moe-runner-backend: "flashinfer_trtllm" + linear-attn-decode-backend: "flashinfer" + + disaggregation-mode: "decode" + disable-radix-cache: true + + mem-fraction-static: 0.8 + context-length: 9236 + max-total-tokens: 1500000 + max-mamba-cache-size: 256 + max-running-requests: 128 + cuda-graph-max-bs: 256 + chunked-prefill-size: 32768 + max-prefill-tokens: 32768 + scheduler-recv-interval: 10 + stream-interval: 30 + page-size: 64 + watchdog-timeout: 1000000 + decode-log-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1x4x8x16x32x64x256" + req_rate: "inf" + random_range_ratio: 0.8 + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + # 9401 is already bound by the cluster-level exporter on im-gb300 nodes; + # use a port outside that range. + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-5p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b4096-c2048-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-5p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b4096-c2048-stp.yaml new file mode 100644 index 0000000000..87f2142aee --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-5p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b4096-c2048-stp.yaml @@ -0,0 +1,190 @@ +# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 5P1D wide-EP +# Prefill: 5 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) +# Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) +# Total: 36 GB300 GPUs (5*4 + 4*4); 8k1k concurrency 1024/2048/3072. +# +# Values taken from ni_experiment_config of pareto row +# qwen3.5-dep16-fia2a-tbo-cc1024x2048x3072-dynamo-tot-nixl +# (sa-qwen-3.5-8k1k-fp4-baseline-mid-pareto study). + +schema: 2 +name: "gb300-fp4-qwen3.5_8k1k_maxtpt_0" + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + source: + pypi: "1.1.0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 2 + nginx_container: nginx + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 5 + workers: 5 + env: + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_ENABLE_SPEC_V2: "1" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_ENABLE_JIT_DEEPGEMM: "true" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" + FLASHINFER_DISABLE_VERSION_CHECK: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_ENABLE_NIXL: "1" + + args: + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + model-path: "/model/" + trust-remote-code: true + + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + disaggregation-transfer-backend: "nixl" + + mem-fraction-static: 0.8 + max-total-tokens: 128000 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + log-level: "info" + page-size: 64 + + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + linear-attn-decode-backend: "flashinfer" + + decode: + nodes: 4 + workers: 1 + + env: + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_ENABLE_SPEC_V2: "1" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_ENABLE_JIT_DEEPGEMM: "true" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" + SGLANG_MOE_NVFP4_DISPATCH: "1" + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: "1" + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: "1" + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: "1" + FLASHINFER_DISABLE_VERSION_CHECK: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + SGLANG_ENABLE_NIXL: "1" + + args: + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + model-path: "/model/" + trust-remote-code: true + + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + enable-two-batch-overlap: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + disaggregation-transfer-backend: "nixl" + + chunked-prefill-size: 4096 + max-mamba-cache-size: 4096 + max-total-tokens: 2200000 + max-running-requests: 4096 + mem-fraction-static: 0.8 + watchdog-timeout: 1000000 + page-size: 64 + + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_cutedsl" + moe-a2a-backend: "flashinfer" + disable-shared-experts-fusion: true + linear-attn-decode-backend: "flashinfer" + + decode-log-interval: 50 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" + random_range_ratio: 0.8 + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml new file mode 100644 index 0000000000..a6352aecb7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml @@ -0,0 +1,188 @@ +# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 6P1D wide-EP +# Prefill: 6 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) +# Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) +# Total: 40 GB300 GPUs (6*4 + 4*4); 8k1k concurrency 5120. +# +# Values taken from ni_experiment_config of pareto row +# qwen3.5-6p_dep4x1d_dep16-fia2a-tbo-cc5120-dynamo-tot-mooncake +# (sa-qwen-3.5-8k1k-fp4-baseline-mid-pareto study). + +schema: 2 +name: "gb300-fp4-qwen3.5_8k1k_maxtpt_1" + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + source: + pypi: "1.1.0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 2 + nginx_container: nginx + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 6 + workers: 6 + env: + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_ENABLE_SPEC_V2: "1" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_ENABLE_JIT_DEEPGEMM: "true" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" + FLASHINFER_DISABLE_VERSION_CHECK: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + model-path: "/model/" + trust-remote-code: true + + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + disaggregation-transfer-backend: "mooncake" + + mem-fraction-static: 0.8 + max-total-tokens: 128000 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + log-level: "info" + page-size: 64 + + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + linear-attn-decode-backend: "flashinfer" + + decode: + nodes: 4 + workers: 1 + + env: + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_ENABLE_SPEC_V2: "1" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_ENABLE_JIT_DEEPGEMM: "true" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" + SGLANG_MOE_NVFP4_DISPATCH: "1" + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: "1" + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: "1" + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: "1" + FLASHINFER_DISABLE_VERSION_CHECK: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + model-path: "/model/" + trust-remote-code: true + + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + enable-two-batch-overlap: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + disaggregation-transfer-backend: "mooncake" + + chunked-prefill-size: 5120 + max-mamba-cache-size: 5120 + max-total-tokens: 3200000 + max-running-requests: 5120 + mem-fraction-static: 0.8 + watchdog-timeout: 1000000 + page-size: 64 + + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_cutedsl" + moe-a2a-backend: "flashinfer" + disable-shared-experts-fusion: true + linear-attn-decode-backend: "flashinfer" + + decode-log-interval: 50 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "5120" + req_rate: "inf" + random_range_ratio: 0.8 + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml new file mode 100644 index 0000000000..942e00dd8d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml @@ -0,0 +1,188 @@ +# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 7P1D wide-EP +# Prefill: 7 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) +# Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) +# Total: 44 GB300 GPUs (7*4 + 4*4); 8k1k concurrency 5120. +# +# Values taken from ni_experiment_config of pareto row +# qwen3.5-7p_dep4x1d_dep16-fia2a-tbo-cc5120-dynamo-tot-mooncake +# (sa-qwen-3.5-8k1k-fp4-baseline-mid-pareto study). + +schema: 2 +name: "gb300-fp4-qwen3.5_8k1k_maxtpt_2" + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + source: + pypi: "1.1.0" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 2 + nginx_container: nginx + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 7 + workers: 7 + env: + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_ENABLE_SPEC_V2: "1" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_ENABLE_JIT_DEEPGEMM: "true" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" + FLASHINFER_DISABLE_VERSION_CHECK: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + + args: + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + model-path: "/model/" + trust-remote-code: true + + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + disaggregation-transfer-backend: "mooncake" + + mem-fraction-static: 0.8 + max-total-tokens: 128000 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + log-level: "info" + page-size: 64 + + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + linear-attn-decode-backend: "flashinfer" + + decode: + nodes: 4 + workers: 1 + + env: + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + SGLANG_ENABLE_SPEC_V2: "1" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_ENABLE_JIT_DEEPGEMM: "true" + SGLANG_ENABLE_FLASHINFER_GEMM: "true" + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" + SGLANG_MOE_NVFP4_DISPATCH: "1" + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: "1" + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: "1" + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: "1" + FLASHINFER_DISABLE_VERSION_CHECK: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" + model-path: "/model/" + trust-remote-code: true + + quantization: "modelopt_fp4" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + enable-two-batch-overlap: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + disaggregation-transfer-backend: "mooncake" + + chunked-prefill-size: 5120 + max-mamba-cache-size: 5120 + max-total-tokens: 3200000 + max-running-requests: 5120 + mem-fraction-static: 0.8 + watchdog-timeout: 1000000 + page-size: 64 + + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_cutedsl" + moe-a2a-backend: "flashinfer" + disable-shared-experts-fusion: true + linear-attn-decode-backend: "flashinfer" + + decode-log-interval: 50 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "5120" + req_rate: "inf" + random_range_ratio: 0.8 + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml new file mode 100644 index 0000000000..4715f76302 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml @@ -0,0 +1,90 @@ +schema: 2 +name: agg-gb300-tp2-c1-mtp-hicache-jid2530006 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 +frontend: + type: sglang-router +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml new file mode 100644 index 0000000000..183c309540 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml @@ -0,0 +1,90 @@ +schema: 2 +name: agg-gb300-tp2-c24-mtp-hicache-jid2530012 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 +frontend: + type: sglang-router +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml new file mode 100644 index 0000000000..099bd88795 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml @@ -0,0 +1,90 @@ +schema: 2 +name: agg-gb300-tp2-c32-mtp-hicache-jid2530013 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 +frontend: + type: sglang-router +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml new file mode 100644 index 0000000000..c03748d22d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml @@ -0,0 +1,90 @@ +schema: 2 +name: agg-gb300-tp2-c40-mtp-hicache-jid2530015 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 +frontend: + type: sglang-router +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml new file mode 100644 index 0000000000..7a5a8417a6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831 +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 +frontend: + type: sglang-router + enable_multiple_frontends: false +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/flashinfer-cache + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-dense-tp-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + moe-a2a-backend: none + load-balance-method: round_robin + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 1536 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 6 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + speculative-moe-runner-backend: flashinfer_trtllm + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + prefill-decode-interval: 0 + mem-fraction-static: 0.85 + max-running-requests: 1 + pp-max-micro-batch-size: 1 + prefill-max-requests: 1 + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-size: 32 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true + cuda-graph-max-bs-decode: 1 + cuda-graph-bs-decode: + - 1 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml new file mode 100644 index 0000000000..ad5b4c86e7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml @@ -0,0 +1,135 @@ +schema: 2 +name: agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831 +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 +frontend: + type: sglang-router + enable_multiple_frontends: false +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: + /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: + /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/flashinfer-cache + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-dense-tp-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + moe-a2a-backend: none + load-balance-method: round_robin + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 1536 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 6 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + speculative-moe-runner-backend: flashinfer_trtllm + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + prefill-decode-interval: 0 + mem-fraction-static: 0.85 + max-running-requests: 2 + pp-max-micro-batch-size: 2 + prefill-max-requests: 2 + cuda-graph-max-bs-decode: 2 + cuda-graph-bs-decode: + - 1 + - 2 + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-size: 128 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml new file mode 100644 index 0000000000..267950162c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml @@ -0,0 +1,90 @@ +schema: 2 +name: agg-gb300-tp2-c48-mtp-hicache-jid2530017 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 +frontend: + type: sglang-router +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml new file mode 100644 index 0000000000..2e3c7cf469 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml @@ -0,0 +1,90 @@ +schema: 2 +name: agg-gb300-tp2-c52-mtp-hicache-jid2527406 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 +frontend: + type: sglang-router +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml new file mode 100644 index 0000000000..3582977cd9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml @@ -0,0 +1,90 @@ +schema: 2 +name: agg-gb300-tp2-c64-mtp-hicache-jid2527410 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 +frontend: + type: sglang-router +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml new file mode 100644 index 0000000000..bc672a8342 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831 +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +frontend: + type: sglang-router + enable_multiple_frontends: false +engine: sglang +roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/c7-mrr1/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/c7-mrr1/flashinfer-cache + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/c7-mrr1/sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-dense-tp-size: 8 + enable-dp-attention: false + enable-dp-lm-head: false + moe-a2a-backend: none + load-balance-method: round_robin + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 1536 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 6 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + speculative-moe-runner-backend: flashinfer_cutedsl + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + mem-fraction-static: 0.85 + max-running-requests: 1 + pp-max-micro-batch-size: 1 + prefill-max-requests: 1 + cuda-graph-max-bs-decode: 1 + cuda-graph-bs-decode: + - 1 + disable-prefill-cuda-graph: true + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-size: 32 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + TP: '8' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml new file mode 100644 index 0000000000..4ac3876eff --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml @@ -0,0 +1,217 @@ +schema: 2 +name: disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + max-mamba-cache-size: 320 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + mamba-max-states-per-path: 1 + decode: + nodes: 1 + workers: 1 + gpus: 2 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 80 + max-running-requests: 80 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + decode-log-interval: 10 + mamba-max-states-per-path: -1 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml new file mode 100644 index 0000000000..11862f0c46 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml @@ -0,0 +1,212 @@ +schema: 2 +name: disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + decode: + nodes: 1 + workers: 1 + gpus: 4 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 160 + watchdog-timeout: 1000000 + decode-log-interval: 10 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml new file mode 100644 index 0000000000..1f6c325744 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml @@ -0,0 +1,212 @@ +schema: 2 +name: disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + decode: + nodes: 1 + workers: 1 + gpus: 4 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml new file mode 100644 index 0000000000..2e293731c7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml @@ -0,0 +1,212 @@ +schema: 2 +name: disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + decode: + nodes: 1 + workers: 1 + gpus: 4 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml new file mode 100644 index 0000000000..5cd6de2ddd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml @@ -0,0 +1,212 @@ +schema: 2 +name: disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + decode: + nodes: 1 + workers: 1 + gpus: 4 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 80 + watchdog-timeout: 1000000 + decode-log-interval: 10 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml new file mode 100644 index 0000000000..405152ad75 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml @@ -0,0 +1,212 @@ +schema: 2 +name: disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + decode: + nodes: 1 + workers: 1 + gpus: 4 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml new file mode 100644 index 0000000000..c61c81724e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml @@ -0,0 +1,212 @@ +schema: 2 +name: disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +dynamo: + install: true + source: + rev: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + decode: + nodes: 1 + workers: 1 + gpus: 4 + env: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + decode-log-interval: 10 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml new file mode 100644 index 0000000000..e790701576 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml @@ -0,0 +1,250 @@ +schema: 2 +name: disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831 +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 + precision: fp4 +dynamo: + install: true + source: + rev: 61c37bc33c3e88da3facaa39fa2b5caf616740ba +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-replica-sync: true + router-queue-threshold: None + router-temperature: '1.0' +engine: sglang +roles: + prefill: + nodes: 3 + workers: 3 + gpus: 4 + env: + # The 0831 image contains the PP+spec runtime merged by SGLang #35758, + # but still carries the pre-merge PP+spec assert in validation_hook.py. + PYTHONOPTIMIZE: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-upstream-main/prefill-deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-upstream-main/prefill-flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '8192' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_PP_LAYER_PARTITION: 16,16,16,12 + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_FLASHINFER_AUTOTUNE_EXTEND: '1' + SGLANG_CACHE_DIR: /tmp/agentx-upstream-main/prefill-sglang-cache + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 1 + pipeline-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + enable-symm-mem: false + disable-overlap-schedule: true + enable-dynamic-chunking: false + mamba-ssm-dtype: bfloat16 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 1048576 + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1536 + max-running-requests: 128 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + speculative-algorithm: NEXTN + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + linear-attn-decode-backend: triton + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + watchdog-timeout: 1000000 + log-level: info + nccl-port: 29500 + scheduler-recv-interval: 1 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + moe-dense-tp-size: 1 + disable-shared-experts-fusion: true + moe-a2a-backend: none + speculative-moe-a2a-backend: none + speculative-moe-runner-backend: flashinfer_trtllm + pp-async-batch-depth: 1 + disable-cuda-graph: true + decode: + nodes: 2 + workers: 2 + gpus: 4 + env: + SGLANG_USE_SYMM_MEM_DP_SYNC: 'true' + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '4' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-upstream-main/decode-deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-upstream-main/decode-flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_CACHE_DIR: /tmp/agentx-upstream-main/decode-sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + data-parallel-size: 4 + expert-parallel-size: 4 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: round_robin + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + moe-a2a-backend: flashinfer + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + speculative-draft-model-quantization: nvfp4_online + speculative-moe-runner-backend: flashinfer_trtllm_routed + speculative-moe-a2a-backend: flashinfer + chunked-prefill-size: 4096 + mem-fraction-static: 0.7 + max-mamba-cache-size: 468 + max-running-requests: 320 + cuda-graph-max-bs: 80 + disaggregation-decode-extra-slots: 2 + stream-interval: 30 + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 32 + decode-log-interval: 30 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml new file mode 100644 index 0000000000..9146587211 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml @@ -0,0 +1,250 @@ +schema: 2 +name: disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831 +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 + precision: fp4 +dynamo: + install: true + source: + rev: 61c37bc33c3e88da3facaa39fa2b5caf616740ba +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 1 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-replica-sync: true + router-queue-threshold: None + router-temperature: '1.0' +engine: sglang +roles: + prefill: + nodes: 3 + workers: 3 + gpus: 4 + env: + # The 0831 image contains the PP+spec runtime merged by SGLang #35758, + # but still carries the pre-merge PP+spec assert in validation_hook.py. + PYTHONOPTIMIZE: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '8192' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_PP_LAYER_PARTITION: 16,16,16,12 + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_FLASHINFER_AUTOTUNE_EXTEND: '1' + SGLANG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-sglang-cache + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 1 + pipeline-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + enable-symm-mem: false + disable-overlap-schedule: true + enable-dynamic-chunking: false + mamba-ssm-dtype: bfloat16 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 1048576 + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1536 + max-running-requests: 128 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + speculative-algorithm: NEXTN + speculative-num-steps: 4 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 5 + linear-attn-decode-backend: triton + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + watchdog-timeout: 1000000 + log-level: info + nccl-port: 29500 + scheduler-recv-interval: 1 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + moe-dense-tp-size: 1 + disable-shared-experts-fusion: true + moe-a2a-backend: none + speculative-moe-a2a-backend: none + speculative-moe-runner-backend: flashinfer_trtllm + pp-async-batch-depth: 1 + disable-cuda-graph: true + decode: + nodes: 4 + workers: 4 + gpus: 4 + env: + SGLANG_USE_SYMM_MEM_DP_SYNC: 'true' + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '4' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/decode-deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-followup/c565-cutedsl-g3072/decode-flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/decode-sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + data-parallel-size: 4 + expert-parallel-size: 4 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: round_robin + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + moe-a2a-backend: flashinfer + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 4 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 5 + speculative-draft-model-quantization: nvfp4_online + speculative-moe-runner-backend: flashinfer_trtllm_routed + speculative-moe-a2a-backend: flashinfer + chunked-prefill-size: 4096 + mem-fraction-static: 0.7 + max-mamba-cache-size: 468 + max-running-requests: 320 + cuda-graph-max-bs: 80 + disaggregation-decode-extra-slots: 2 + stream-interval: 30 + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 32 + decode-log-interval: 30 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..82dde3eea7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml @@ -0,0 +1,167 @@ +schema: 2 +name: "qwen3.5-1p1d-dep4-dep16" + + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 4096 + mem-fraction-static: 0.80 + max-mamba-cache-size: 2048 + max-running-requests: 2048 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "512x1024x2048" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml new file mode 100644 index 0000000000..606c99861e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml @@ -0,0 +1,136 @@ +schema: 2 +name: "qwen3.5-1p1d-tp4-tp4" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 4096 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "prefill" + + decode: + nodes: 1 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_HEALTH_CHECK_TIMEOUT: "3600" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 4096 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "decode" + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 10 + num_warmup_mult: 1 + random_range_ratio: 0.8 + concurrencies: "1x2x4x8x16x32x64" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..32d37132f1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml @@ -0,0 +1,167 @@ +schema: 2 +name: "qwen3.5-2p1d-dep4-dep16" + + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 2 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 2 + workers: 2 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 8192 + mem-fraction-static: 0.75 + max-mamba-cache-size: 4096 + max-running-requests: 4096 + cuda-graph-max-bs: 256 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "4096" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b1024-c1x2x8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b1024-c1x2x8-mtp.yaml new file mode 100644 index 0000000000..9050a22244 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b1024-c1x2x8-mtp.yaml @@ -0,0 +1,156 @@ +# Qwen3.5 FP8 GB300 disaggregated MTP 1P1D TP4/TP4 configuration. + +schema: 2 +name: "qwen3.5-fp8-gb300-mtp-8k1k-1p1d-tp4-tp4" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 16384 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "prefill" + + decode: + nodes: 1 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + max-mamba-cache-size: 1024 + moe-runner-backend: "flashinfer_trtllm" + + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 16384 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "decode" + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "1x2x8" + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml new file mode 100644 index 0000000000..8ce688a8ba --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml @@ -0,0 +1,170 @@ +# Qwen3.5 FP8 GB300 disaggregated MTP 1P1D TEP8/TEP8 configuration. + +schema: 2 +name: "qwen3.5-fp8-gb300-mtp-8k1k-1p1d-tep8-tep8" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 2 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + max-total-tokens: 128000 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 9236 + cuda-graph-max-bs: 320 + scheduler-recv-interval: 10 + decode-log-interval: 50 + stream-interval: 50 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "mooncake" + + decode: + nodes: 2 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + max-mamba-cache-size: 1024 + moe-runner-backend: "flashinfer_trtllm" + + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + max-total-tokens: 2200000 + chunked-prefill-size: 4096 + max-prefill-tokens: 16384 + context-length: 9236 + cuda-graph-max-bs: 320 + scheduler-recv-interval: 10 + decode-log-interval: 50 + stream-interval: 50 + disaggregation-mode: "decode" + disaggregation-transfer-backend: "mooncake" + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 20 + num_warmup_mult: 2 + random_range_ratio: 0.8 + concurrencies: "32x48x80" + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml new file mode 100644 index 0000000000..2bc2fb0378 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml @@ -0,0 +1,136 @@ +schema: 2 +name: "qwen3.5-1p1d-tp4-tp4" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 1 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 16384 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "prefill" + + decode: + nodes: 1 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_HEALTH_CHECK_TIMEOUT: "3600" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + attention-backend: "trtllm_mha" + tensor-parallel-size: 4 + mamba-ssm-dtype: "bfloat16" + moe-runner-backend: "flashinfer_trtllm" + disable-radix-cache: true + max-running-requests: 1024 + mem-fraction-static: 0.8 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + context-length: 16384 + cuda-graph-max-bs: 1024 + decode-log-interval: 1 + stream-interval: 50 + disaggregation-mode: "decode" + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "1x2x4x8x16x32x64x128" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + # 9401 is already bound by the cluster-level exporter on im-gb300 nodes; + # use a port outside that range. + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml new file mode 100644 index 0000000000..e4588b3dda --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml @@ -0,0 +1,186 @@ +# Qwen3.5 FP8 GB300 disaggregated MTP 3P1D DEP4/DEP16 configuration. + +schema: 2 +name: "qwen3.5-fp8-gb300-mtp-8k1k-3p1d-dep4-dep16" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 2 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 3 + workers: 3 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.7 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 16384 + mem-fraction-static: 0.7 + max-mamba-cache-size: 1024 + max-running-requests: 1024 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 20 + num_warmup_mult: 2 + random_range_ratio: 0.8 + concurrencies: "480" + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..aaf9afce75 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml @@ -0,0 +1,172 @@ +schema: 2 +name: "qwen3.5-4p1d-dep4-dep16" + + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 3 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 4 + workers: 4 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 16384 + mem-fraction-static: 0.80 + max-mamba-cache-size: 2048 + max-running-requests: 2048 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 20 + num_warmup_mult: 2 + random_range_ratio: 0.8 + concurrencies: "1024" + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml new file mode 100644 index 0000000000..e0cc1ab0a3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml @@ -0,0 +1,186 @@ +# Qwen3.5 FP8 GB300 disaggregated MTP 4P1D DEP4/DEP16 configuration. + +schema: 2 +name: "qwen3.5-fp8-gb300-mtp-8k1k-4p1d-dep4-dep16" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 3 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 4 + workers: 4 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.7 + chunked-prefill-size: 98304 + max-prefill-tokens: 24576 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 16384 + mem-fraction-static: 0.7 + max-mamba-cache-size: 1024 + max-running-requests: 1024 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 20 + num_warmup_mult: 2 + random_range_ratio: 0.8 + concurrencies: "768" + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml new file mode 100644 index 0000000000..510e6146a5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml @@ -0,0 +1,186 @@ +# Qwen3.5 FP8 GB300 disaggregated MTP 6P1D DEP4/DEP16 configuration. + +schema: 2 +name: "qwen3.5-fp8-gb300-mtp-8k1k-6p1d-dep4-dep16" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 5 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 6 + workers: 6 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31001 + mem-fraction-static: 0.7 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31001 + + chunked-prefill-size: 4096 + context-length: 16384 + mem-fraction-static: 0.7 + max-mamba-cache-size: 1024 + max-running-requests: 1024 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 20 + num_warmup_mult: 2 + random_range_ratio: 0.8 + concurrencies: "1280" + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml new file mode 100644 index 0000000000..0759135190 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml @@ -0,0 +1,186 @@ +# Qwen3.5 FP8 GB300 disaggregated MTP 7P1D DEP4/DEP16 configuration. + +schema: 2 +name: "qwen3.5-fp8-gb300-mtp-8k1k-7p1d-dep4-dep16" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 6 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 7 + workers: 7 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31001 + mem-fraction-static: 0.7 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31001 + + chunked-prefill-size: 4096 + context-length: 16384 + mem-fraction-static: 0.7 + max-mamba-cache-size: 2048 + max-running-requests: 2048 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 20 + num_warmup_mult: 2 + random_range_ratio: 0.8 + concurrencies: "1344" + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-8p1d-dep4-dep16-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-8p1d-dep4-dep16-stp.yaml new file mode 100644 index 0000000000..e52fd2beb7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-8p1d-dep4-dep16-stp.yaml @@ -0,0 +1,168 @@ +schema: 2 +name: "qwen3.5-8p1d-dep4-dep16" + + +sbatch_directives: + mem: "0" + +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 8 + workers: 8 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + + chunked-prefill-size: 4096 + context-length: 16384 + mem-fraction-static: 0.80 + max-mamba-cache-size: 2048 + max-running-requests: 2048 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + random_range_ratio: 0.8 + concurrencies: "2048x4096" diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml new file mode 100644 index 0000000000..505b8cef0e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml @@ -0,0 +1,186 @@ +# Qwen3.5 FP8 GB300 disaggregated MTP 8P1D DEP4/DEP16 configuration. + +schema: 2 +name: "qwen3.5-fp8-gb300-mtp-8k1k-8p1d-dep4-dep16" + +sbatch_directives: + mem: "0" + +dynamo: + install: true + + source: + rev: 46520ca59afe992fb5ef61b3197b2316f8df9b2b +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 7 + nginx_container: nginx + +model: + path: "qwen3.5-fp8" + container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" + precision: "fp8" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 +engine: sglang +roles: + prefill: + nodes: 8 + workers: 8 + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + trust-remote-code: true + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 2048 + mamba-ssm-dtype: "bfloat16" + disaggregation-mode: "prefill" + disable-radix-cache: true + disaggregation-bootstrap-port: 31001 + mem-fraction-static: 0.7 + chunked-prefill-size: 98304 + max-prefill-tokens: 24576 + load-balance-method: "round_robin" + watchdog-timeout: 1000000 + disable-cuda-graph: true + log-level: "info" + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "flashinfer_trtllm" + + decode: + nodes: 4 + workers: 1 + + env: + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_SPEC_V2: "1" + NO_COLOR: "1" + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" + TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" + PYTHONUNBUFFERED: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TE_METRIC: "true" + SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" + FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" + SGLANG_HEALTH_CHECK_TIMEOUT: "1800" + SGLANG_HEALTH_STARTING_OK: "1" + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" + + args: + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + model-path: "/model/" + random-seed: 42 + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + + mamba-scheduler-strategy: "no_buffer" + mamba-track-interval: 128 + mamba-ssm-dtype: "bfloat16" + + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + disaggregation-mode: "decode" + disable-radix-cache: true + disaggregation-bootstrap-port: 31001 + + chunked-prefill-size: 4096 + context-length: 16384 + mem-fraction-static: 0.7 + max-mamba-cache-size: 2048 + max-running-requests: 2048 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + + page-size: 64 + attention-backend: "trtllm_mha" + moe-runner-backend: "deep_gemm" + moe-a2a-backend: "deepep" + deepep-mode: "low_latency" + ep-dispatch-algorithm: "static" + eplb-algorithm: "deepseek" + + decode-log-interval: 1 + stream-interval: 50 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: "inf" + num_prompts_mult: 20 + num_warmup_mult: 2 + random_range_ratio: 0.8 + concurrencies: "1920x2304" + use_chat_template: true + +telemetry: + enabled: true + collect_interval_ms: 1000 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp0-c2150.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp0-c2150.yaml new file mode 100644 index 0000000000..da222ed0ff --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp0-c2150.yaml @@ -0,0 +1,166 @@ +schema: 2 +name: disagg-gb300-10p1d-dep1-dep8-c2150-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + sequential_node_start: 2 +roles: + prefill: + nodes: 3 + workers: 10 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 2 + workers: 1 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 256 + max_num_tokens: 256 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 2150 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-11p1d-dep16-b64-eplb0-mtp0-c1076.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-11p1d-dep16-b64-eplb0-mtp0-c1076.yaml new file mode 100644 index 0000000000..e8ae211d16 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-11p1d-dep16-b64-eplb0-mtp0-c1076.yaml @@ -0,0 +1,144 @@ +schema: 2 +name: disagg-gb300-11p1d-dep1-dep16-c1076-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + sequential_node_start: 2 +roles: + prefill: + nodes: 3 + workers: 11 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 64 + max_num_tokens: 64 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 1076 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-11p1d-dep8-b128-eplb0-mtp3-c1229.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-11p1d-dep8-b128-eplb0-mtp3-c1229.yaml new file mode 100644 index 0000000000..e3c25c88ac --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-11p1d-dep8-b128-eplb0-mtp3-c1229.yaml @@ -0,0 +1,173 @@ +schema: 2 +name: disagg-gb300-11p1d-dep1-dep8-c1229-mtp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + sequential_node_start: 2 +roles: + prefill: + nodes: 3 + workers: 11 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 2 + workers: 1 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + - 20 + - 24 + - 28 + - 32 + - 36 + - 40 + - 44 + - 48 + - 52 + - 56 + - 60 + - 64 + - 68 + - 72 + - 76 + - 80 + - 84 + - 88 + - 92 + - 96 + - 100 + - 104 + - 108 + - 112 + - 116 + - 120 + - 124 + - 128 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 128 + max_num_tokens: 512 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 1229 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-16p1d-dep16-b128-eplb0-mtp0-c2253.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-16p1d-dep16-b128-eplb0-mtp0-c2253.yaml new file mode 100644 index 0000000000..b804ea9c6f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-16p1d-dep16-b128-eplb0-mtp0-c2253.yaml @@ -0,0 +1,152 @@ +schema: 2 +name: disagg-gb300-16p1d-dep1-dep16-c2253-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + sequential_node_start: 2 +roles: + prefill: + nodes: 4 + workers: 16 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 128 + max_num_tokens: 128 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 2253 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-17p2d-dep8-b64-eplb0-mtp3-c1126.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-17p2d-dep8-b64-eplb0-mtp3-c1126.yaml new file mode 100644 index 0000000000..9b9c31e8d4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-17p2d-dep8-b64-eplb0-mtp3-c1126.yaml @@ -0,0 +1,155 @@ +schema: 2 +name: disagg-gb300-17p2d-dep1-dep8-c1126-mtp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + sequential_node_start: 2 +roles: + prefill: + nodes: 5 + workers: 17 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 4 + workers: 2 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + - 20 + - 24 + - 28 + - 32 + - 36 + - 40 + - 44 + - 48 + - 52 + - 56 + - 60 + - 64 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + max_batch_size: 64 + max_num_tokens: 256 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 1126 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p2d-tep8-b16-eplb0-mtp0-c42.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p2d-tep8-b16-eplb0-mtp0-c42.yaml new file mode 100644 index 0000000000..5bf05178f4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p2d-tep8-b16-eplb0-mtp0-c42.yaml @@ -0,0 +1,132 @@ +schema: 2 +name: disagg-gb300-1p2d-dep1-tep8-c42-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 4 + workers: 2 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9472 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 42 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p2d-tep8-b8-eplb0-mtp3-c20.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p2d-tep8-b8-eplb0-mtp3-c20.yaml new file mode 100644 index 0000000000..bcc1ba4916 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p2d-tep8-b8-eplb0-mtp3-c20.yaml @@ -0,0 +1,137 @@ +schema: 2 +name: disagg-gb300-1p2d-dep1-tep8-c20-mtp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 4 + workers: 2 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9472 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 20 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0-c8.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0-c8.yaml new file mode 100644 index 0000000000..25d2d802d3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0-c8.yaml @@ -0,0 +1,128 @@ +schema: 2 +name: disagg-gb300-1p4d-dep2-tep8-c8-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 2 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 2 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 2 + trust_remote_code: true + decode: + nodes: 8 + workers: 4 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9472 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 8 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3-c12.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3-c12.yaml new file mode 100644 index 0000000000..1f6e44e916 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3-c12.yaml @@ -0,0 +1,136 @@ +schema: 2 +name: disagg-gb300-1p4d-dep1-tep8-c12-mtp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 8 + workers: 4 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 1 + max_num_tokens: 4 + max_seq_len: 9472 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 12 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b2-eplb0-mtp3-c8.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b2-eplb0-mtp3-c8.yaml new file mode 100644 index 0000000000..40d27d0507 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b2-eplb0-mtp3-c8.yaml @@ -0,0 +1,138 @@ +schema: 2 +name: disagg-gb300-1p4d-dep1-tep8-c8-mtp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 8 + workers: 4 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 8 + max_seq_len: 9472 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 8 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml new file mode 100644 index 0000000000..0c6d5b3cc7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml @@ -0,0 +1,130 @@ +schema: 2 +name: disagg-gb300-1p4d-dep1-tep8-c24-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: trtllm +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 8 + workers: 4 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 4 + max_num_tokens: 4 + max_seq_len: 9472 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 24 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-24p1d-dep16-b512-eplb0-mtp0-c8192-mtp-sweep.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-24p1d-dep16-b512-eplb0-mtp0-c8192-mtp-sweep.yaml new file mode 100644 index 0000000000..410e07db36 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-24p1d-dep16-b512-eplb0-mtp0-c8192-mtp-sweep.yaml @@ -0,0 +1,198 @@ +schema: 2 +name: disagg-gb300-24p1d-dep1-dep16-c8192-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + sequential_node_start: 2 +roles: + prefill: + nodes: 6 + workers: 24 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + - 264 + - 272 + - 280 + - 288 + - 296 + - 304 + - 312 + - 320 + - 328 + - 336 + - 344 + - 352 + - 360 + - 368 + - 376 + - 384 + - 392 + - 400 + - 408 + - 416 + - 424 + - 432 + - 440 + - 448 + - 456 + - 464 + - 472 + - 480 + - 488 + - 496 + - 504 + - 512 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 512 + max_num_tokens: 512 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 8192 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-24p1d-dep16-b512-eplb0-mtp0-c8192-stp-sweep.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-24p1d-dep16-b512-eplb0-mtp0-c8192-stp-sweep.yaml new file mode 100644 index 0000000000..410e07db36 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-24p1d-dep16-b512-eplb0-mtp0-c8192-stp-sweep.yaml @@ -0,0 +1,198 @@ +schema: 2 +name: disagg-gb300-24p1d-dep1-dep16-c8192-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + sequential_node_start: 2 +roles: + prefill: + nodes: 6 + workers: 24 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + - 264 + - 272 + - 280 + - 288 + - 296 + - 304 + - 312 + - 320 + - 328 + - 336 + - 344 + - 352 + - 360 + - 368 + - 376 + - 384 + - 392 + - 400 + - 408 + - 416 + - 424 + - 432 + - 440 + - 448 + - 456 + - 464 + - 472 + - 480 + - 488 + - 496 + - 504 + - 512 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 512 + max_num_tokens: 512 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 8192 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-26p1d-dep16-b256-eplb0-mtp2-c4301.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-26p1d-dep16-b256-eplb0-mtp2-c4301.yaml new file mode 100644 index 0000000000..f16e546399 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-26p1d-dep16-b256-eplb0-mtp2-c4301.yaml @@ -0,0 +1,205 @@ +schema: 2 +name: disagg-gb300-26p1d-dep1-dep16-c4301-mtp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + sequential_node_start: 2 +roles: + prefill: + nodes: 7 + workers: 26 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 2 + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + - 20 + - 24 + - 28 + - 32 + - 36 + - 40 + - 44 + - 48 + - 52 + - 56 + - 60 + - 64 + - 68 + - 72 + - 76 + - 80 + - 84 + - 88 + - 92 + - 96 + - 100 + - 104 + - 108 + - 112 + - 116 + - 120 + - 124 + - 128 + - 132 + - 136 + - 140 + - 144 + - 148 + - 152 + - 156 + - 160 + - 164 + - 168 + - 172 + - 176 + - 180 + - 184 + - 188 + - 192 + - 196 + - 200 + - 204 + - 208 + - 212 + - 216 + - 220 + - 224 + - 228 + - 232 + - 236 + - 240 + - 244 + - 248 + - 252 + - 256 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 256 + max_num_tokens: 768 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 2 + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 4301 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep16-b16-eplb0-mtp0-c282.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep16-b16-eplb0-mtp0-c282.yaml new file mode 100644 index 0000000000..04390fc340 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep16-b16-eplb0-mtp0-c282.yaml @@ -0,0 +1,136 @@ +schema: 2 +name: disagg-gb300-4p1d-dep1-dep16-c282-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + sequential_node_start: 2 +roles: + prefill: + nodes: 1 + workers: 4 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 282 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-4p3d-tep8-b32-eplb0-mtp3-c126.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-4p3d-tep8-b32-eplb0-mtp3-c126.yaml new file mode 100644 index 0000000000..5e35f10696 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-4p3d-tep8-b32-eplb0-mtp3-c126.yaml @@ -0,0 +1,145 @@ +schema: 2 +name: disagg-gb300-4p3d-dep1-tep8-c126-mtp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + sequential_node_start: 2 +roles: + prefill: + nodes: 1 + workers: 4 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 6 + workers: 3 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + - 20 + - 24 + - 28 + - 32 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9472 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 126 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-4p3d-tep8-b64-eplb0-mtp0-c210.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-4p3d-tep8-b64-eplb0-mtp0-c210.yaml new file mode 100644 index 0000000000..4cc6f60a02 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-4p3d-tep8-b64-eplb0-mtp0-c210.yaml @@ -0,0 +1,142 @@ +schema: 2 +name: disagg-gb300-4p3d-dep1-tep8-c210-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + sequential_node_start: 2 +roles: + prefill: + nodes: 1 + workers: 4 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 6 + workers: 3 + gpus: 8 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 64 + max_num_tokens: 64 + max_seq_len: 9472 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 210 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-5p1d-dep16-b8-eplb0-mtp3-c154.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-5p1d-dep16-b8-eplb0-mtp3-c154.yaml new file mode 100644 index 0000000000..8bc5d4e327 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-5p1d-dep16-b8-eplb0-mtp3-c154.yaml @@ -0,0 +1,143 @@ +schema: 2 +name: disagg-gb300-5p1d-dep1-dep16-c154-mtp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + sequential_node_start: 2 +roles: + prefill: + nodes: 2 + workers: 5 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 8 + max_num_tokens: 32 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 154 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep16-b32-eplb0-mtp0-c563.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep16-b32-eplb0-mtp0-c563.yaml new file mode 100644 index 0000000000..3f430da1ce --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep16-b32-eplb0-mtp0-c563.yaml @@ -0,0 +1,138 @@ +schema: 2 +name: disagg-gb300-7p1d-dep1-dep16-c563-stp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + sequential_node_start: 2 +roles: + prefill: + nodes: 2 + workers: 7 + gpus: 1 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + CTX_LOAD_STAGGER_S: '180' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + max_batch_size: 2 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 1 + trust_remote_code: true + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 563 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep16-b32-eplb0-mtp3-c666.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep16-b32-eplb0-mtp3-c666.yaml new file mode 100644 index 0000000000..ade5667c13 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep16-b32-eplb0-mtp3-c666.yaml @@ -0,0 +1,143 @@ +schema: 2 +name: disagg-gb300-7p1d-dep2-dep16-c666-mtp +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + wheel: 1.4.0.dev20260807 + +identity: + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + sequential_node_start: 2 +roles: + prefill: + nodes: 4 + workers: 7 + gpus: 2 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + max_batch_size: 32 + max_num_tokens: 16896 + max_seq_len: 8448 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 2 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 2 + trust_remote_code: true + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + args: + cache_transceiver_config: + backend: UCX + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 16384 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 12 + - 16 + - 20 + - 24 + - 28 + - 32 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.75 + max_batch_size: 32 + max_num_tokens: 128 + max_seq_len: 9472 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 100 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' + DYN_TOKENIZER: "fastokens" + +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 666 + req_rate: inf + num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml new file mode 100644 index 0000000000..57c70d7921 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml @@ -0,0 +1,224 @@ +schema: 2 +name: disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 + +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 1 + env: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + args: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: true + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 1 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + nodes: 1 + workers: 1 + gpus: 2 + env: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: true + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 8 + max_num_tokens: 56 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 2 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 2 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml new file mode 100644 index 0000000000..a1788e0775 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml @@ -0,0 +1,221 @@ +schema: 2 +name: disagg-gb300-1p7d-dep4-tep8-c7-b1-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 + +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '4.04' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + args: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: true + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 4 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + nodes: 14 + workers: 7 + gpus: 8 + env: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '4.04' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: true + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 1 + max_num_tokens: 8 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 7 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml new file mode 100644 index 0000000000..458f220d02 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml @@ -0,0 +1,223 @@ +schema: 2 +name: disagg-gb300-2p2d-dep1-tep2-c52-b4-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 + +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 1 + env: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + args: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: true + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 1 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + nodes: 1 + workers: 2 + gpus: 2 + env: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: true + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 4 + max_num_tokens: 28 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 2 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 2 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml new file mode 100644 index 0000000000..4e81c377ef --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml @@ -0,0 +1,238 @@ +schema: 2 +name: disagg-gb300-2p3d-tep2-tep8-c96-b128-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 + +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false +roles: + prefill: + nodes: 1 + workers: 2 + gpus: 2 + env: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + args: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: false + enable_attention_dp: false + enable_chunked_prefill: true + enable_iter_perf_stats: true + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 16384 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 2 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 2 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + - 16384 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + nodes: 6 + workers: 3 + gpus: 8 + env: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: true + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 128 + max_num_tokens: 896 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml new file mode 100644 index 0000000000..2cc82a5125 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml @@ -0,0 +1,222 @@ +schema: 2 +name: disagg-gb300-3p1d-dep4-dep16-c565-b8-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 + +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false +roles: + prefill: + nodes: 3 + workers: 3 + gpus: 4 + env: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + args: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: true + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 4 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: true + enable_iter_perf_stats: true + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 8 + max_num_tokens: 56 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml new file mode 100644 index 0000000000..6ea1813945 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml @@ -0,0 +1,225 @@ +schema: 2 +name: disagg-gb300-3p2d-dep4-dep4-c704-b32-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + source: + git: https://github.com/cquil11/dynamo.git + rev: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 + +slurm: + time_limit: 04:00:00 +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 +engine: + type: trtllm + publish_events_and_metrics: false +roles: + prefill: + nodes: 3 + workers: 3 + gpus: 4 + env: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + args: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: true + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 4 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + nodes: 2 + workers: 2 + gpus: 4 + env: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + args: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: true + enable_iter_perf_stats: true + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 32 + max_num_tokens: 224 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8.yaml deleted file mode 100644 index c47004151a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8.yaml +++ /dev/null @@ -1,154 +0,0 @@ -name: "disagg-b200-1p1d-dep8-dep8-c512" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" - precision: "fp4" - -dynamo: - hash: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -health_check: - max_attempts: 240 -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256x512" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8.yaml deleted file mode 100644 index 2a4c75da52..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8.yaml +++ /dev/null @@ -1,146 +0,0 @@ -name: "disagg-b200-1p1d-dep8-tp8-4-c64" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" - precision: "fp4" - -dynamo: - hash: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -health_check: - max_attempts: 240 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_TOPK_V2: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "32x64" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p1d-tp8-tp8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p1d-tp8-tp8.yaml deleted file mode 100644 index 457bb7ef47..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p1d-tp8-tp8.yaml +++ /dev/null @@ -1,123 +0,0 @@ -name: "disagg-b200-1p1d-tp8-tp8-c1" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" - precision: "fp4" - -dynamo: - hash: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -health_check: - max_attempts: 240 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.90 - max-running-requests: 1024 - cuda-graph-max-bs: 512 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8.yaml deleted file mode 100644 index b5143d53e0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8.yaml +++ /dev/null @@ -1,155 +0,0 @@ -name: "disagg-b200-1p2d-dep8-dep8-c256" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" - precision: "fp4" - -dynamo: - hash: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 2 - decode_workers: 2 - gpus_per_decode: 8 - -health_check: - max_attempts: 240 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p4d-dep8-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p4d-dep8-dep8.yaml deleted file mode 100644 index e550c817a1..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p4d-dep8-dep8.yaml +++ /dev/null @@ -1,155 +0,0 @@ -name: "disagg-b200-1p4d-dep8-dep8-c256" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" - precision: "fp4" - -dynamo: - hash: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 4 - gpus_per_decode: 8 - -health_check: - max_attempts: 240 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p4d-dep8-tp8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p4d-dep8-tp8.yaml deleted file mode 100644 index b5c4fd468a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p4d-dep8-tp8.yaml +++ /dev/null @@ -1,146 +0,0 @@ -name: "disagg-b200-1p4d-dep8-tp8-c64" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" - precision: "fp4" - -dynamo: - hash: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 4 - gpus_per_decode: 8 - -health_check: - max_attempts: 240 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_TOPK_V2: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "64" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-3p2d-dep8-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-3p2d-dep8-dep8.yaml deleted file mode 100644 index 773d2f2b44..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-3p2d-dep8-dep8.yaml +++ /dev/null @@ -1,155 +0,0 @@ -name: "disagg-b200-3p2d-dep8-dep8-c2048" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" - precision: "fp4" - -dynamo: - hash: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 3 - prefill_workers: 3 - gpus_per_prefill: 8 - decode_nodes: 2 - decode_workers: 2 - gpus_per_decode: 8 - -health_check: - max_attempts: 240 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-4p2d-dep8-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-4p2d-dep8-dep8.yaml deleted file mode 100644 index d30449d293..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-4p2d-dep8-dep8.yaml +++ /dev/null @@ -1,155 +0,0 @@ -name: "disagg-b200-4p2d-dep8-dep8-c4096" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" - precision: "fp4" - -dynamo: - hash: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 4 - prefill_workers: 4 - gpus_per_prefill: 8 - decode_nodes: 2 - decode_workers: 2 - gpus_per_decode: 8 - -health_check: - max_attempts: 240 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-5p2d-dep8-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-5p2d-dep8-dep8.yaml deleted file mode 100644 index f15029ed9e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-5p2d-dep8-dep8.yaml +++ /dev/null @@ -1,155 +0,0 @@ -name: "disagg-b200-5p2d-dep8-dep8-c6144" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" - precision: "fp4" - -dynamo: - hash: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 5 - prefill_workers: 5 - gpus_per_prefill: 8 - decode_nodes: 2 - decode_workers: 2 - gpus_per_decode: 8 - -health_check: - max_attempts: 240 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "6144" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-6p2d-dep8-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-6p2d-dep8-dep8.yaml deleted file mode 100644 index bd85ebe9fb..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-6p2d-dep8-dep8.yaml +++ /dev/null @@ -1,155 +0,0 @@ -name: "disagg-b200-6p2d-dep8-dep8-c8192" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260821-f825d729" - precision: "fp4" - -dynamo: - hash: "86f84b9486905c2bfaa4b756c9fa39f04ecf5ebd" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 6 - prefill_workers: 6 - gpus_per_prefill: 8 - decode_nodes: 2 - decode_workers: 2 - gpus_per_decode: 8 - -health_check: - max_attempts: 240 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml deleted file mode 100644 index 2e2016a15b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml +++ /dev/null @@ -1,133 +0,0 @@ -name: "disagg-b200-8k1k-low-latency-1p1d-tp8-tp8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -health_check: - max_attempts: 240 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - NCCL_CUMEM_ENABLE: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - NCCL_CUMEM_ENABLE: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 16 - cuda-graph-max-bs: 8 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 8 - cuda-graph-max-bs: 8 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml deleted file mode 100644 index bfb83c48dd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml +++ /dev/null @@ -1,140 +0,0 @@ -name: "disagg-b200-8k1k-low-latency-1p6d-dep8-tp8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 6 - decode_workers: 6 - gpus_per_decode: 8 - -health_check: - max_attempts: 240 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_CUMEM_ENABLE: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_CUMEM_ENABLE: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.9 - max-running-requests: 256 - cuda-graph-max-bs: 128 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 128 - cuda-graph-max-bs: 128 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "32x64x128" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml deleted file mode 100644 index b2291ca000..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml +++ /dev/null @@ -1,141 +0,0 @@ -name: "disagg-b200-8k1k-mid-curve-1p1d-dep8-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -health_check: - max_attempts: 240 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - NCCL_CUMEM_ENABLE: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - NCCL_CUMEM_ENABLE: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - mem-fraction-static: 0.8 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - mem-fraction-static: 0.9 - max-running-requests: 1536 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "256x1024" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml deleted file mode 100644 index e709c508c8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml +++ /dev/null @@ -1,152 +0,0 @@ -name: "disagg-b200-8k1k-mid-curve-1p2d-dep8-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 2 - decode_workers: 2 - gpus_per_decode: 8 - -health_check: - max_attempts: 240 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_CUMEM_ENABLE: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_CUMEM_ENABLE: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.8 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 65536 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.85 - max-running-requests: 1024 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "256" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml deleted file mode 100644 index b1f2e1eba7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml +++ /dev/null @@ -1,141 +0,0 @@ -name: "disagg-b200-8k1k-mid-curve-5p3d-dep8-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 5 - prefill_workers: 5 - gpus_per_prefill: 8 - decode_nodes: 3 - decode_workers: 3 - gpus_per_decode: 8 - -health_check: - max_attempts: 240 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - NCCL_CUMEM_ENABLE: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - NCCL_CUMEM_ENABLE: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - mem-fraction-static: 0.8 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - mem-fraction-static: 0.9 - max-running-requests: 2560 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "6144" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index 3ea1d3e722..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,147 +0,0 @@ -name: "disagg-b300-1p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - mem-fraction-static: 0.9 - max-running-requests: 256 - cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.94 - max-running-requests: 3072 - cuda-graph-max-bs: 256 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "256" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8.yaml deleted file mode 100644 index 42bd30e552..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8.yaml +++ /dev/null @@ -1,158 +0,0 @@ -name: "disagg-b300-1p1d-dep4-dep8" - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18432 - cuda-graph-max-bs: 1280 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4-mtp.yaml deleted file mode 100644 index 2293fa9a81..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4-mtp.yaml +++ /dev/null @@ -1,138 +0,0 @@ -name: "disagg-b300-1p1d-tp4-tp4-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 4 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 8 - cuda-graph-max-bs: 8 - chunked-prefill-size: 32768 - - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 8 - cuda-graph-max-bs: 8 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4.yaml deleted file mode 100644 index e55d8dc2f5..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4.yaml +++ /dev/null @@ -1,133 +0,0 @@ -name: "disagg-b300-1p1d-tp4-tp4" - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.90 - max-running-requests: 32 - cuda-graph-max-bs: 32 - chunked-prefill-size: 32768 - - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 64 - cuda-graph-max-bs: 64 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1x8x16x32" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p2d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p2d-dep4-dep8-mtp.yaml deleted file mode 100644 index 5737b5d734..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p2d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: "disagg-b300-1p2d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 2 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 256 - cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.94 - max-running-requests: 3072 - cuda-graph-max-bs: 256 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "256" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p6d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p6d-dep4-tp4-mtp.yaml deleted file mode 100644 index fe5819b0af..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p6d-dep4-tp4-mtp.yaml +++ /dev/null @@ -1,142 +0,0 @@ -name: "disagg-b300-1p6d-dep4-tp4-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 3 - decode_workers: 6 - gpus_per_decode: 4 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 128 - cuda-graph-max-bs: 128 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "8x32x64" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index 96912af606..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: "disagg-b300-2p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 2 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 3072 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "512" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8.yaml deleted file mode 100644 index 4c5c5ef4db..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8.yaml +++ /dev/null @@ -1,158 +0,0 @@ -name: "disagg-b300-2p1d-dep4-dep8" - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 2 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18432 - cuda-graph-max-bs: 1280 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "768" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index 2b9713b708..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: "disagg-b300-4p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 2 - prefill_workers: 4 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 3072 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1024" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8.yaml deleted file mode 100644 index 8ad309af71..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8.yaml +++ /dev/null @@ -1,158 +0,0 @@ -name: "disagg-b300-4p1d-dep4-dep8" - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 2 - prefill_workers: 4 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18432 - cuda-graph-max-bs: 1280 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index f7ff198115..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: "disagg-b300-6p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 3 - prefill_workers: 6 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - - mem-fraction-static: 0.9 - max-running-requests: 4096 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 9216 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "2048" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8.yaml deleted file mode 100644 index 6ba7519714..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8.yaml +++ /dev/null @@ -1,158 +0,0 @@ -name: "disagg-b300-6p1d-dep4-dep8" - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 3 - prefill_workers: 6 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: nixl - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: nixl - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18432 - cuda-graph-max-bs: 1280 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "3072" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-8p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-8p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index 7bdc43dd73..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-8p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: "disagg-b300-8p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "41882ae9b07232eed4850fb1daf8c958abb2556a" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 4 - prefill_workers: 8 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - UCX_TLS: "cuda_copy,rc" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - watchdog-timeout: 86400 - disaggregation-transfer-backend: nixl - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 2 - - mem-fraction-static: 0.85 - max-running-requests: 8192 - cuda-graph-max-bs: 1280 - swa-full-tokens-ratio: 0.1 - context-length: 9216 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "3072" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-dep8-dep16-6-c512.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-dep8-dep16-6-c512.yaml deleted file mode 100644 index 153369da5b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-dep8-dep16-6-c512.yaml +++ /dev/null @@ -1,169 +0,0 @@ -name: "disagg-gb200-1p1d-dep8-dep16-6-c512" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "512" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-tp8-tp8-4-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-tp8-tp8-4-c1.yaml deleted file mode 100644 index f3b4416313..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-tp8-tp8-4-c1.yaml +++ /dev/null @@ -1,130 +0,0 @@ -name: "disagg-gb200-1p1d-tp8-tp8-4-c1" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.90 - max-running-requests: 1024 - cuda-graph-max-bs: 512 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p2d-dep8-dep16-10-c256.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p2d-dep8-dep16-10-c256.yaml deleted file mode 100644 index 79fa253f78..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p2d-dep8-dep16-10-c256.yaml +++ /dev/null @@ -1,169 +0,0 @@ -name: "disagg-gb200-1p2d-dep8-dep16-10-c256" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 8 - decode_workers: 2 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p4d-dep8-tp8-10-c64.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p4d-dep8-tp8-10-c64.yaml deleted file mode 100644 index 38373f90dc..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p4d-dep8-tp8-10-c64.yaml +++ /dev/null @@ -1,155 +0,0 @@ -name: "disagg-gb200-1p4d-dep8-tp8-10-c64" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 8 - decode_workers: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_TOPK_V2: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "64" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-2p1d-dep8-dep16-8-c1536.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-2p1d-dep8-dep16-8-c1536.yaml deleted file mode 100644 index f2f8a5b259..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-2p1d-dep8-dep16-8-c1536.yaml +++ /dev/null @@ -1,169 +0,0 @@ -name: "disagg-gb200-2p1d-dep8-dep16-8-c1536" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 4 - prefill_workers: 2 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1536" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-4p1d-dep8-dep16-12-c4096.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-4p1d-dep8-dep16-12-c4096.yaml deleted file mode 100644 index 0e64f4ab2f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-4p1d-dep8-dep16-12-c4096.yaml +++ /dev/null @@ -1,169 +0,0 @@ -name: "disagg-gb200-4p1d-dep8-dep16-12-c4096" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 8 - prefill_workers: 4 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-5p1d-dep8-dep16-14-c8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-5p1d-dep8-dep16-14-c8192.yaml deleted file mode 100644 index 32139590b6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-5p1d-dep8-dep16-14-c8192.yaml +++ /dev/null @@ -1,169 +0,0 @@ -name: "disagg-gb200-5p1d-dep8-dep16-14-c8192" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 10 - prefill_workers: 5 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-6p1d-dep8-dep12-15-c8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-6p1d-dep8-dep12-15-c8192.yaml deleted file mode 100644 index 769d95c2b0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-6p1d-dep8-dep12-15-c8192.yaml +++ /dev/null @@ -1,169 +0,0 @@ -name: "disagg-gb200-6p1d-dep8-dep12-15-c8192" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 12 - prefill_workers: 6 - gpus_per_prefill: 8 - decode_nodes: 3 - decode_workers: 1 - gpus_per_decode: 12 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 12 - data-parallel-size: 12 - expert-parallel-size: 12 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" - use_chat_template: false - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p1d-tp8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p1d-tp8-tp8-mtp.yaml deleted file mode 100644 index 1a96cd064e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p1d-tp8-tp8-mtp.yaml +++ /dev/null @@ -1,137 +0,0 @@ -name: "dsv4-pro-gb200-disagg-8k1k-low-latency-1p1d-tp8-tp8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 16 - cuda-graph-max-bs: 8 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 8 - cuda-graph-max-bs: 8 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p6d-dep8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p6d-dep8-tp8-mtp.yaml deleted file mode 100644 index 124c146032..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p6d-dep8-tp8-mtp.yaml +++ /dev/null @@ -1,144 +0,0 @@ -name: "dsv4-pro-gb200-disagg-8k1k-low-latency-1p6d-dep8-tp8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 12 - decode_workers: 6 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.9 - max-running-requests: 256 - cuda-graph-max-bs: 128 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 128 - cuda-graph-max-bs: 128 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "32x64x128" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-1p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-1p1d-dep8-dep16-mtp.yaml deleted file mode 100644 index 36d766894a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-1p1d-dep8-dep16-mtp.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-1p1d-dep8-dep16-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.8 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 65536 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.85 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1024" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-2p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-2p1d-dep8-dep16-mtp.yaml deleted file mode 100644 index 292e7bf698..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-2p1d-dep8-dep16-mtp.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-2p1d-dep8-dep16-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 4 - prefill_workers: 2 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.8 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.85 - max-running-requests: 2048 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "2048" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-3p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-3p1d-dep8-dep16-mtp.yaml deleted file mode 100644 index 961cedbf5b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-3p1d-dep8-dep16-mtp.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-3p1d-dep8-dep16-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 6 - prefill_workers: 3 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.8 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.85 - max-running-requests: 4096 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "3072" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-4p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-4p1d-dep8-dep16-mtp.yaml deleted file mode 100644 index 033feac2e9..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-4p1d-dep8-dep16-mtp.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-4p1d-dep8-dep16-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 8 - prefill_workers: 4 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.8 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.85 - max-running-requests: 6144 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "6144" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-5p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-5p1d-dep8-dep16-mtp.yaml deleted file mode 100644 index 1de31484a2..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-5p1d-dep8-dep16-mtp.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-5p1d-dep8-dep16-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 10 - prefill_workers: 5 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.8 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.85 - max-running-requests: 16384 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "8192" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-6p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-6p1d-dep8-dep16-mtp.yaml deleted file mode 100644 index 1e7c89d0f4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-6p1d-dep8-dep16-mtp.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-6p1d-dep8-dep16-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 12 - prefill_workers: 6 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.8 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.85 - max-running-requests: 21504 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "16384" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml deleted file mode 100644 index 6b2bd71f3c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml +++ /dev/null @@ -1,168 +0,0 @@ -name: "disagg-gb300-10p1d-dep4-dep32-18-c2500" - -# Weiliang wide-EP sweep point: EP=32, 10P+8D = 18 nodes, conc=2500. -# Matches srt-slurm PR#173 zip_override EP=32 topology. -# Env vars and sglang_config from InferenceX main (not Weiliang's 0510 image). - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp4" - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 10 - prefill_workers: 10 - gpus_per_prefill: 4 - decode_nodes: 8 - decode_workers: 1 - gpus_per_decode: 32 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 32 - data-parallel-size: 32 - expert-parallel-size: 32 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18432 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "2500" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml deleted file mode 100644 index eedb49a2ec..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml +++ /dev/null @@ -1,168 +0,0 @@ -name: "disagg-gb300-12p1d-dep4-dep24-18-c3000" - -# Weiliang wide-EP sweep point: EP=24, 12P+6D = 18 nodes, conc=3000. -# Matches srt-slurm PR#173 zip_override EP=24 topology. -# Env vars and sglang_config from InferenceX main (not Weiliang's 0510 image). - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp4" - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 12 - prefill_workers: 12 - gpus_per_prefill: 4 - decode_nodes: 6 - decode_workers: 1 - gpus_per_decode: 24 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 24 - data-parallel-size: 24 - expert-parallel-size: 24 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18432 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "3000" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml deleted file mode 100644 index faffc09419..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml +++ /dev/null @@ -1,168 +0,0 @@ -name: "disagg-gb300-14p1d-dep4-dep16-18-c8192" - -# Weiliang wide-EP sweep point: EP=16, 14P+4D = 18 nodes, conc=8192. -# Matches srt-slurm PR#173 zip_override EP=16 topology. -# Env vars and sglang_config from InferenceX main (not Weiliang's 0510 image). - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp4" - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 14 - prefill_workers: 14 - gpus_per_prefill: 4 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18432 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml deleted file mode 100644 index 36d8c25966..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml +++ /dev/null @@ -1,168 +0,0 @@ -name: "disagg-gb300-15p1d-dep4-dep12-18-c12000" - -# Weiliang wide-EP sweep point: EP=12, 15P+3D = 18 nodes, conc=12000. -# Matches srt-slurm PR#173 zip_override EP=12 topology. -# Env vars and sglang_config from InferenceX main (not Weiliang's 0510 image). - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp4" - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 15 - prefill_workers: 15 - gpus_per_prefill: 4 - decode_nodes: 3 - decode_workers: 1 - gpus_per_decode: 12 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 12 - data-parallel-size: 12 - expert-parallel-size: 12 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18432 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "12000" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml deleted file mode 100644 index 8cd19d06b6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml +++ /dev/null @@ -1,194 +0,0 @@ -name: "disagg-gb300-1p1d-dep4-dep16-5-c1024" - -# 8k/1k high-throughput topology for the wideep DSV4-Pro setup. -# -# Schema/values come from PR #1213 (513cbef) — that PR introduced the -# `dsv4-pro-gb300-fp4` upstream-style recipe with two `zip_override` -# variants (wideep [0] / narrow_ep [1]) and `backend.benchmark`. Our -# pinned srtctl (NVIDIA/srt-slurm @ sa-submission-q2-2026) doesn't -# support either: `zip_override_*_hightpt` rejects with `Unknown field` -# and `benchmark` only validates at top level. So this file inlines the -# wideep [0] override and lifts `benchmark` back out — same operational -# values, schema the pinned srtctl will accept. -# -# Other adjustments back to the InferenceX cluster shape: container & -# model.path restored to the aliases mapped in launch_gb300.sh's -# srtslurm.yaml (`lmsysorg/sglang:deepseek-v4-grace-blackwell` and -# `deepseek-v4-pro`); `dynamo.install: true` added so the container -# (which has no dynamo baked in) installs from the pinned hash. -# -# Cluster-specific items NOT inlined (require InferenceX-side equivalents): -# - slurm.partition (the source cluster uses `hpc-mid`) -# - frontend.nginx_container (yangminl's `nginx-1.27.4.sqsh` path) -# - extra_mount: yangminl/sglang-patched/sglang. Earlier diff analysis -# showed only `expert_location_dispatch.py` topk_ids int32 cast is an -# active runtime diff vs container sglang; other patched files are -# env-gated dead code under the same SGLANG_OPT_* flags this yaml -# already sets. -# -# DG-related env intentionally diverged (DG cache path is host-specific): -# - SGLANG_DG_CACHE_DIR=/configs/deepgemm_cache (yangminl host) -# - SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 (yangminl uses prebuilt cache) -# This yaml uses SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 instead. - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp4" - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - # is single-node only and corrupts results in 2-node decode setups. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - mem-fraction-static: 0.90 - max-running-requests: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1024" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - # 9401 is already bound by the cluster-level exporter on im-gb300 nodes; - # use a port outside that range. - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml deleted file mode 100644 index 3818eb3ef5..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml +++ /dev/null @@ -1,177 +0,0 @@ -name: "disagg-gb300-1p1d-tp4-tp4-2-c1" - -# 8k/1k high-throughput topology for the wideep DSV4-Pro setup. -# -# Schema/values come from PR #1213 (513cbef) — that PR introduced the -# `dsv4-pro-gb300-fp4` upstream-style recipe with two `zip_override` -# variants (wideep [0] / narrow_ep [1]) and `backend.benchmark`. Our -# pinned srtctl (NVIDIA/srt-slurm @ sa-submission-q2-2026) doesn't -# support either: `zip_override_*_hightpt` rejects with `Unknown field` -# and `benchmark` only validates at top level. So this file inlines the -# wideep [0] override and lifts `benchmark` back out — same operational -# values, schema the pinned srtctl will accept. -# -# Other adjustments back to the InferenceX cluster shape: container & -# model.path restored to the aliases mapped in launch_gb300.sh's -# srtslurm.yaml (`lmsysorg/sglang:deepseek-v4-grace-blackwell` and -# `deepseek-v4-pro`); `dynamo.install: true` added so the container -# (which has no dynamo baked in) installs from the pinned hash. -# -# Cluster-specific items NOT inlined (require InferenceX-side equivalents): -# - slurm.partition (the source cluster uses `hpc-mid`) -# - frontend.nginx_container (yangminl's `nginx-1.27.4.sqsh` path) -# - extra_mount: yangminl/sglang-patched/sglang. Earlier diff analysis -# showed only `expert_location_dispatch.py` topk_ids int32 cast is an -# active runtime diff vs container sglang; other patched files are -# env-gated dead code under the same SGLANG_OPT_* flags this yaml -# already sets. -# -# DG-related env intentionally diverged (DG cache path is host-specific): -# - SGLANG_DG_CACHE_DIR=/configs/deepgemm_cache (yangminl host) -# - SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 (yangminl uses prebuilt cache) -# This yaml uses SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 instead. - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp4" - -# See ../1k1k/disagg-gb200-1p1d-dep8-tep8.yaml for the dynamo pin -# rationale. Hash bumped from PR #1213 to track the dynamo-sglang dsv4 -# dev branch. -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -# Match yangminl's working all-dynamo.yaml on the source cluster: -# cpus-per-task=144 — without this slurm hands out 1 CPU/task, which -# turns the dynamo `hash:` cold source build (~500 rust crates, -# ravif/exr/zip/pyo3 stack) into a 30+ min serial compile. With 144 -# cargo finishes in ~5 min. -# mem=0 — slurm's "give the whole node's memory"; needed -# for sglang loading 671B FP4 weights + dynamo build at the same -# time without OOM. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -# Topology: 7 prefill (TP=4 / DP=4 / EP=4 / 1 node each) + 1 decode -# (TP=8 / DP=8 / EP=8 / 2 nodes). 9 nodes total. -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - # is single-node only and corrupts results in 2-node decode setups. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - # 9401 is already bound by the cluster-level exporter on im-gb300 nodes; - # use a port outside that range. - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml deleted file mode 100644 index dac81ce20f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml +++ /dev/null @@ -1,169 +0,0 @@ -name: "disagg-gb300-8p1d-dep4-dep40-18-c2048" - -# Weiliang wide-EP sweep point: EP=40, 8P+10D = 18 nodes, conc=2048. -# Matches srt-slurm PR#173 zip_override EP=40 topology. -# Env vars and sglang_config from InferenceX main (not Weiliang's 0510 image). - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp4" - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 8 - prefill_workers: 8 - gpus_per_prefill: 4 - decode_nodes: 10 - decode_workers: 1 - gpus_per_decode: 40 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 40 - data-parallel-size: 40 - expert-parallel-size: 40 - ep-num-redundant-experts: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18400 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index c01462f482..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,154 +0,0 @@ -name: "dsv4-pro-gb300-disagg-8k1k-high-conc-6p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" - precision: "mxfp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 6 - prefill_workers: 6 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - - mem-fraction-static: 0.9 - max-running-requests: 4096 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 9216 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "4096" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index 32e29c59fd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,154 +0,0 @@ -name: "dsv4-pro-gb300-disagg-8k1k-high-conc-8p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" - precision: "mxfp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 8 - prefill_workers: 8 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 2 - - mem-fraction-static: 0.85 - max-running-requests: 8192 - cuda-graph-max-bs: 1280 - swa-full-tokens-ratio: 0.1 - context-length: 9216 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "8192" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml deleted file mode 100644 index 1981b131a9..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml +++ /dev/null @@ -1,134 +0,0 @@ -name: "dsv4-pro-gb300-disagg-8k1k-low-latency-1p1d-tp4-tp4-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" - precision: "mxfp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - decode_nodes: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 - # is single-node only and corrupts results in 2-node decode setups. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 8 - cuda-graph-max-bs: 8 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 8 - cuda-graph-max-bs: 8 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml deleted file mode 100644 index 7d9642799d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "dsv4-pro-gb300-disagg-8k1k-low-latency-1p6d-dep4-tp4-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" - precision: "mxfp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - decode_nodes: 6 - decode_workers: 6 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 - # is single-node only and corrupts results in 2-node decode setups. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 128 - cuda-graph-max-bs: 128 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "8x32x64" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml deleted file mode 100644 index 7fba5c9158..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml +++ /dev/null @@ -1,154 +0,0 @@ -name: "dsv4-pro-gb300-disagg-8k1k-mid-curve-1p1d-dep4-dep16-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" - precision: "mxfp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 256 - cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.94 - max-running-requests: 3072 - cuda-graph-max-bs: 256 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "256" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index 0211764426..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,154 +0,0 @@ -name: "dsv4-pro-gb300-disagg-8k1k-mid-curve-1p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" - precision: "mxfp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 256 - cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.94 - max-running-requests: 3072 - cuda-graph-max-bs: 256 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "256" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index f722c73f83..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,154 +0,0 @@ -name: "dsv4-pro-gb300-disagg-8k1k-mid-curve-2p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" - precision: "mxfp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 2 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 3072 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "512" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index a609d54cb0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,154 +0,0 @@ -name: "dsv4-pro-gb300-disagg-8k1k-mid-curve-4p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" - precision: "mxfp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 4 - prefill_workers: 4 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 3072 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1024" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml deleted file mode 100644 index 3d3e53696f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml +++ /dev/null @@ -1,121 +0,0 @@ -name: "agg-gb300-tp4-mtp-lowlatency" - -# Low-latency AgentX aggregate topology: one TP4 worker occupies one -# four-GPU GB300 node and serves both prefill and decode with DSpark K=6. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9" - -dynamo: - install: true - wheel: "1.5.0.dev20260910" - -slurm: - time_limit: "4:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - aggregated_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: high - PIP_BREAK_SYSTEM_PACKAGES: "1" - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - - sglang_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" - enable-metrics: true - enable-cache-report: true - trust-remote-code: true - weight-loader-prefetch-checkpoints: true - stream-interval: 10 - watchdog-timeout: 1000000 - mem-fraction-static: 0.94 - page-size: 256 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - moe-runner-backend: "flashinfer_mxfp4" - enable-deepseek-v4-fp4-indexer: true - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 32 - cuda-graph-max-bs-decode: 32 - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 4 - ep-size: 1 - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "false" - TP: "4" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml deleted file mode 100644 index 0a728df926..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml +++ /dev/null @@ -1,121 +0,0 @@ -name: "agg-gb300-tp8-mtp-lowlatency" - -# Low-latency AgentX aggregate topology: one TP8 worker spans two -# four-GPU GB300 nodes and serves both prefill and decode with DSpark K=6. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9" - -dynamo: - install: true - wheel: "1.5.0.dev20260910" - -slurm: - time_limit: "4:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - aggregated_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: high - PIP_BREAK_SYSTEM_PACKAGES: "1" - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - - sglang_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" - enable-metrics: true - enable-cache-report: true - trust-remote-code: true - weight-loader-prefetch-checkpoints: true - stream-interval: 10 - watchdog-timeout: 1000000 - mem-fraction-static: 0.94 - page-size: 256 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - moe-runner-backend: "flashinfer_mxfp4" - enable-deepseek-v4-fp4-indexer: true - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 4 - cuda-graph-max-bs-decode: 4 - scheduler-recv-interval: 30 - dp-size: 1 - tp-size: 8 - ep-size: 1 - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "false" - TP: "8" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml deleted file mode 100644 index fb12502bd6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml +++ /dev/null @@ -1,134 +0,0 @@ -name: "agg-h200-tp8-mtp-kvoffload" - -# DeepSeek-V4-Pro AgentX aggregated TP8 serving on one 8xH200 node. The recipe is -# topology-invariant; configs/nvidia-master.yaml varies only concurrency to -# produce the latency/throughput Pareto curve. - -model: - path: "deepseek-v4-pro" - container: "dynamo-sglang" - precision: "fp8" - -dynamo: - install: true - wheel: "1.3.0.dev20260718" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "h200" - gpus_per_node: 8 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 8 - -infra: - etcd_nats_dedicated_node: true - # AgentX prompts can exceed NATS's 1 MiB default after JSON serialization. - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - aggregated_environment: - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_SIMULATE_ACC_LEN: "2.49" - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - PIP_BREAK_SYSTEM_PACKAGES: "1" - HF_HUB_CACHE: /hf_hub_cache - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "0" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_TOPK_V2: "True" - SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" - - sglang_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - enable-metrics: true - trust-remote-code: true - stream-interval: 50 - watchdog-timeout: 1000000 - mem-fraction-static: 0.88 - chunked-prefill-size: 4096 - moe-runner-backend: marlin - disable-flashinfer-autotune: true - swa-full-tokens-ratio: 0.1 - max-running-requests: 32 - cuda-graph-max-bs: 32 - - scheduler-recv-interval: 30 - tp-size: 8 - dp-size: 1 - ep-size: 1 - - enable-hierarchical-cache: true - hicache-ratio: 6 - hicache-write-policy: write_through - hicache-io-backend: kernel - - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - -srun_options: - # The custom benchmark installs its isolated AIPerf environment at runtime. - container-remap-root: "" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: false - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - # Aggregated serving uses one TP8 worker for both prefill and decode. - IS_MULTINODE: "true" - TP: "8" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - # Fail the run loudly if the aggregate engine's sglang: metrics never reach - # the AIPerf export, instead of publishing a trace artifact with no backend - # series behind it. - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload.yaml deleted file mode 100644 index 96c49f4ec3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload.yaml +++ /dev/null @@ -1,225 +0,0 @@ -name: "disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (2P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 480. -# -# Uses the flat single-variant srtctl schema the agentic CI flow expects; -# resources + backend (prefill/decode env + sglang_config) are normalized -# from the Pareto run. -# Concurrency is exported into agentic_srt.sh from the master-config conc-list. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" - -dynamo: - install: true - wheel: "1.5.0.dev20260902" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 16 - -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - prefill_environment: - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - - decode_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - - sglang_config: - prefill: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 256 - cuda-graph-max-bs: 256 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - - decode: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 16 - dp-size: 16 - ep-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 3072 - cuda-graph-max-bs: 256 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload.yaml deleted file mode 100644 index 67570f735c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload.yaml +++ /dev/null @@ -1,227 +0,0 @@ -name: "disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro on GB300 -# (4P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 960. -# -# DEP8-prefill variant aligned with the measured Pareto point: prefill uses -# tp/dp/ep 8, four nodes, and SGLANG_DSV4_MHC_PREWARM=1. -# Concurrency is exported into agentic_srt.sh -# from the master-config conc-list. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" - -dynamo: - install: true - wheel: "1.5.0.dev20260902" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 4 - prefill_workers: 2 - decode_workers: 1 - - -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - prefill_environment: - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - OMP_NUM_THREADS: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_DSV4_MHC_PREWARM: '1' - - decode_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - OMP_NUM_THREADS: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_DSV4_MHC_PREWARM: '1' - - sglang_config: - prefill: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 256 - cuda-graph-max-bs: 256 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - - decode: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 16 - dp-size: 16 - ep-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 3072 - cuda-graph-max-bs: 256 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload.yaml deleted file mode 100644 index 95d1e9205f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload.yaml +++ /dev/null @@ -1,225 +0,0 @@ -name: "disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (6P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1440. -# -# Uses the flat single-variant srtctl schema the agentic CI flow expects; -# resources + backend (prefill/decode env + sglang_config) are normalized -# from the Pareto run. -# Concurrency is exported into agentic_srt.sh from the master-config conc-list. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" - -dynamo: - install: true - wheel: "1.5.0.dev20260902" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 6 - decode_nodes: 4 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 16 - -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - prefill_environment: - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - - decode_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - - sglang_config: - prefill: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - - decode: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 16 - dp-size: 16 - ep-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 3072 - cuda-graph-max-bs: 512 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload.yaml deleted file mode 100644 index 3408902a6a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload.yaml +++ /dev/null @@ -1,230 +0,0 @@ -name: "disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload" - -# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300 -# (8P x DEP8 / 4D x DEP16, DSpark K=6 + hierarchical-cache KV offload), tuned for concurrency 1920. -# -# Uses the flat single-variant srtctl schema the agentic CI flow expects; -# resources + backend (prefill/decode env + sglang_config) are normalized -# from the Pareto run. -# Concurrency is exported into agentic_srt.sh from the master-config conc-list. - -model: - path: "deepseek-v4-pro-0813" - container: "dynamo-sglang" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro-0813" - container: - image: "lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21" - -dynamo: - install: true - wheel: "1.5.0.dev20260902" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 8 - decode_nodes: 4 - prefill_workers: 4 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 16 - -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - nginx_keepalive_timeout: "900s" - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - # AgentX warmup can legitimately keep the single wide decode worker busy - # for longer than Dynamo's 10-second TCP request-plane default. - DYN_TCP_REQUEST_TIMEOUT: "60" - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - prefill_environment: - SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" - SGLANG_DSV4_MHC_PREWARM: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' - SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - - decode_environment: - SGLANG_RAGGED_VERIFY_MODE: "static" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache - SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_DSV4_REASONING_EFFORT: high - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' - SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' - SGLANG_OPT_USE_ONLINE_COMPRESS: '0' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - MC_FORCE_MNNVL: '1' - NCCL_TIMEOUT: '100000' - NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000' - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn - SGLANG_LOG_FORWARD_ITERS: '1' - SGLANG_LOG_MS: '1' - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' - - sglang_config: - prefill: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 8 - dp-size: 8 - ep-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - load-balance-method: total_tokens - mem-fraction-static: 0.85 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - - decode: - host: 0.0.0.0 - served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - watchdog-timeout: 86400 - stream-interval: 60 - tp-size: 16 - dp-size: 16 - ep-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - moe-a2a-backend: megamoe - enable-deepseek-v4-fp4-indexer: true - enable-w4a4-mxfp4-megamoe: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - load-balance-method: total_tokens - mem-fraction-static: 0.9 - page-size: 256 - swa-full-tokens-ratio: 0.02 - max-running-requests: 3072 - cuda-graph-max-bs: 192 - disable-flashinfer-autotune: true - model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' - speculative-algorithm: DSPARK - speculative-dspark-block-size: 6 - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml deleted file mode 100644 index 11f56972ee..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml +++ /dev/null @@ -1,123 +0,0 @@ -# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml -# base + zip_override_mtp_lowlat[0]): 1p5d low-latency (dep4 prefill / tep8 decode, 5 decode nodes). -# One flat YAML per concrete topology, matching the 8k1k local recipe layout -# (sglang//-//disagg//...). - -name: b200-fp4-mtp-low-latency-dep4-1p-tep8-5d -model: - path: dsr1 - container: dynamo-sglang - precision: fp4 - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -resources: - gpu_type: b200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 5 - decode_workers: 5 - gpus_per_node: 8 -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - DYN_REQUEST_PLANE: nats - SGLANG_ENABLE_SPEC_V2: '1' - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - DYN_REQUEST_PLANE: nats - SGLANG_ENABLE_SPEC_V2: '1' - sglang_config: - prefill: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 2200 - max-running-requests: 512 - disable-cuda-graph: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - moe-dense-tp-size: 1 - stream-interval: 30 - watchdog-timeout: 1000000 - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - fp4-gemm-backend: flashinfer_trtllm - decode: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 2200 - max-running-requests: 512 - cuda-graph-max-bs: 512 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 8 - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - stream-interval: 30 - watchdog-timeout: 1000000 - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - fp4-gemm-backend: flashinfer_trtllm - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - req_rate: inf - concurrencies: 16x512 - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml deleted file mode 100644 index acbec6f11a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml +++ /dev/null @@ -1,123 +0,0 @@ -# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml -# base + zip_override_mtp_lowlat[1]): 1p6d low-latency (dep4 prefill / tep8 decode, 6 decode nodes). -# One flat YAML per concrete topology, matching the 8k1k local recipe layout -# (sglang//-//disagg//...). - -name: b200-fp4-mtp-low-latency-dep4-1p-tep8-6d -model: - path: dsr1 - container: dynamo-sglang - precision: fp4 - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -resources: - gpu_type: b200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 6 - decode_workers: 6 - gpus_per_node: 8 -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - DYN_REQUEST_PLANE: nats - SGLANG_ENABLE_SPEC_V2: '1' - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - DYN_REQUEST_PLANE: nats - SGLANG_ENABLE_SPEC_V2: '1' - sglang_config: - prefill: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 2200 - max-running-requests: 512 - disable-cuda-graph: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - moe-dense-tp-size: 1 - stream-interval: 30 - watchdog-timeout: 1000000 - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - fp4-gemm-backend: flashinfer_trtllm - decode: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 2200 - max-running-requests: 512 - cuda-graph-max-bs: 512 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 8 - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - stream-interval: 30 - watchdog-timeout: 1000000 - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - fp4-gemm-backend: flashinfer_trtllm - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - req_rate: inf - concurrencies: 32x64x256x512 - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_0.yaml deleted file mode 100644 index 5f67dfb50f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_0.yaml +++ /dev/null @@ -1,128 +0,0 @@ -# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml -# base + zip_override_mtp_maxtpt[0]): 1p1d max-throughput (dep4 prefill / dep8 decode, mem-fraction 0.75). -# One flat YAML per concrete topology, matching the 8k1k local recipe layout -# (sglang//-//disagg//...). - -name: b200-fp4-mtp-max-tpt-dep4-1p-dep8-1d -model: - path: dsr1 - container: dynamo-sglang - precision: fp4 - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -resources: - gpu_type: b200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_node: 8 -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - DYN_REQUEST_PLANE: nats - SGLANG_ENABLE_SPEC_V2: '1' - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - DYN_REQUEST_PLANE: nats - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_ENABLE_SPEC_V2: '1' - sglang_config: - prefill: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 2200 - max-running-requests: 1024 - disable-cuda-graph: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - moe-dense-tp-size: 1 - stream-interval: 30 - watchdog-timeout: 1000000 - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - fp4-gemm-backend: flashinfer_trtllm - decode: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.75 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 2200 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - stream-interval: 30 - watchdog-timeout: 1000000 - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - fp4-gemm-backend: flashinfer_trtllm - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - req_rate: inf - concurrencies: 512x1024 - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_1.yaml deleted file mode 100644 index 36ceac6122..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_1.yaml +++ /dev/null @@ -1,128 +0,0 @@ -# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml -# base + zip_override_mtp_maxtpt[1]): 1p2d max-throughput (dep4 prefill / dep8 decode, mem-fraction 0.85). -# One flat YAML per concrete topology, matching the 8k1k local recipe layout -# (sglang//-//disagg//...). - -name: b200-fp4-mtp-max-tpt-dep4-1p-dep8-2d -model: - path: dsr1 - container: dynamo-sglang - precision: fp4 - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -resources: - gpu_type: b200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 2 - gpus_per_node: 8 -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - DYN_REQUEST_PLANE: nats - SGLANG_ENABLE_SPEC_V2: '1' - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - DYN_REQUEST_PLANE: nats - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_ENABLE_SPEC_V2: '1' - sglang_config: - prefill: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 2200 - max-running-requests: 512 - disable-cuda-graph: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - moe-dense-tp-size: 1 - stream-interval: 30 - watchdog-timeout: 1000000 - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - fp4-gemm-backend: flashinfer_trtllm - decode: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 2200 - max-running-requests: 512 - cuda-graph-max-bs: 512 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - stream-interval: 30 - watchdog-timeout: 1000000 - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - fp4-gemm-backend: flashinfer_trtllm - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - req_rate: inf - concurrencies: '512' - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_1p1d.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_1p1d.yaml deleted file mode 100644 index 13a1a7d9d0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_1p1d.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "b200-fp4-mtp2-throughput-1p-dep4-1d-dep8" - -# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/b200-fp4/8k1k.yaml -# base + override_mtp2_throughput_1p1d, DEP4 prefill / DEP8 decode, MTP2). -# One flat YAML per concrete topology, matching the InferenceX glm5 disagg -# layout (sglang//-//disagg//...). - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx-sqsh - -model: - path: "dsr1" - container: "lmsysorg/sglang:v0.5.12.post1" - precision: "fp4" - -resources: - gpu_type: "b200" - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_node: 8 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - sglang_config: - prefill: - disaggregation-mode: "prefill" - served-model-name: "deepseek-ai/DeepSeek-R1" - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.95 - max-total-tokens: 32768 - chunked-prefill-size: 24576 - cuda-graph-max-bs: 256 - max-running-requests: 2048 - scheduler-recv-interval: 1 - load-balance-method: "round_robin" - disaggregation-bootstrap-port: null - data-parallel-size: 4 - tensor-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - - decode: - disaggregation-mode: "decode" - served-model-name: "deepseek-ai/DeepSeek-R1" - prefill-round-robin-balance: true - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - disaggregation-bootstrap-port: 30001 - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.85 - chunked-prefill-size: 8192 - cuda-graph-max-bs: 1024 - max-running-requests: 2048 - scheduler-recv-interval: 1 - data-parallel-size: 8 - tensor-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: 300 - concurrencies: "512" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_2p1d.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_2p1d.yaml deleted file mode 100644 index 37225d44e8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_2p1d.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "b200-fp4-mtp2-throughput-2p-dep4-1d-dep8" - -# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/b200-fp4/8k1k.yaml -# base + override_mtp2_throughput_2p1d, DEP4 prefill / DEP8 decode, MTP2). -# One flat YAML per concrete topology, matching the InferenceX glm5 disagg -# layout (sglang//-//disagg//...). - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx-sqsh - -model: - path: "dsr1" - container: "lmsysorg/sglang:v0.5.12.post1" - precision: "fp4" - -resources: - gpu_type: "b200" - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_node: 8 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - sglang_config: - prefill: - disaggregation-mode: "prefill" - served-model-name: "deepseek-ai/DeepSeek-R1" - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.95 - max-total-tokens: 32768 - chunked-prefill-size: 24576 - cuda-graph-max-bs: 256 - max-running-requests: 2048 - scheduler-recv-interval: 1 - load-balance-method: "round_robin" - disaggregation-bootstrap-port: null - data-parallel-size: 4 - tensor-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - - decode: - disaggregation-mode: "decode" - served-model-name: "deepseek-ai/DeepSeek-R1" - prefill-round-robin-balance: true - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - disaggregation-bootstrap-port: 30001 - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.85 - chunked-prefill-size: 8192 - cuda-graph-max-bs: 1024 - max-running-requests: 2048 - scheduler-recv-interval: 1 - data-parallel-size: 8 - tensor-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: 300 - concurrencies: "768" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_3p1d.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_3p1d.yaml deleted file mode 100644 index d8e401620a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_3p1d.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "b200-fp4-mtp2-throughput-3p-dep4-1d-dep8" - -# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/b200-fp4/8k1k.yaml -# base + override_mtp2_throughput_3p1d, DEP4 prefill / DEP8 decode, MTP2). -# One flat YAML per concrete topology, matching the InferenceX glm5 disagg -# layout (sglang//-//disagg//...). - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx-sqsh - -model: - path: "dsr1" - container: "lmsysorg/sglang:v0.5.12.post1" - precision: "fp4" - -resources: - gpu_type: "b200" - prefill_nodes: 2 - decode_nodes: 1 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_node: 8 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - sglang_config: - prefill: - disaggregation-mode: "prefill" - served-model-name: "deepseek-ai/DeepSeek-R1" - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.95 - max-total-tokens: 32768 - chunked-prefill-size: 24576 - cuda-graph-max-bs: 256 - max-running-requests: 2048 - scheduler-recv-interval: 1 - load-balance-method: "round_robin" - disaggregation-bootstrap-port: null - data-parallel-size: 4 - tensor-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - - decode: - disaggregation-mode: "decode" - served-model-name: "deepseek-ai/DeepSeek-R1" - prefill-round-robin-balance: true - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - disaggregation-bootstrap-port: 30001 - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.85 - chunked-prefill-size: 8192 - cuda-graph-max-bs: 1024 - max-running-requests: 2048 - scheduler-recv-interval: 1 - data-parallel-size: 8 - tensor-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: 300 - concurrencies: "1024" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_4p1d.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_4p1d.yaml deleted file mode 100644 index a3923b02db..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_4p1d.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "b200-fp4-mtp2-throughput-4p-dep4-1d-dep8" - -# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/b200-fp4/8k1k.yaml -# base + override_mtp2_throughput_4p1d, DEP4 prefill / DEP8 decode, MTP2). -# One flat YAML per concrete topology, matching the InferenceX glm5 disagg -# layout (sglang//-//disagg//...). - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx-sqsh - -model: - path: "dsr1" - container: "lmsysorg/sglang:v0.5.12.post1" - precision: "fp4" - -resources: - gpu_type: "b200" - prefill_nodes: 2 - decode_nodes: 1 - prefill_workers: 4 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_node: 8 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - sglang_config: - prefill: - disaggregation-mode: "prefill" - served-model-name: "deepseek-ai/DeepSeek-R1" - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.95 - max-total-tokens: 32768 - chunked-prefill-size: 24576 - cuda-graph-max-bs: 256 - max-running-requests: 2048 - scheduler-recv-interval: 1 - load-balance-method: "round_robin" - disaggregation-bootstrap-port: null - data-parallel-size: 4 - tensor-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - - decode: - disaggregation-mode: "decode" - served-model-name: "deepseek-ai/DeepSeek-R1" - prefill-round-robin-balance: true - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - disaggregation-bootstrap-port: 30001 - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.85 - chunked-prefill-size: 8192 - cuda-graph-max-bs: 1024 - max-running-requests: 2048 - scheduler-recv-interval: 1 - data-parallel-size: 8 - tensor-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: 300 - concurrencies: "512" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_5p1d.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_5p1d.yaml deleted file mode 100644 index bcbedcb68b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_5p1d.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "b200-fp4-mtp2-throughput-5p-dep4-1d-dep8" - -# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/b200-fp4/8k1k.yaml -# base + override_mtp2_throughput_5p1d, DEP4 prefill / DEP8 decode, MTP2). -# One flat YAML per concrete topology, matching the InferenceX glm5 disagg -# layout (sglang//-//disagg//...). - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx-sqsh - -model: - path: "dsr1" - container: "lmsysorg/sglang:v0.5.12.post1" - precision: "fp4" - -resources: - gpu_type: "b200" - prefill_nodes: 3 - decode_nodes: 1 - prefill_workers: 5 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_node: 8 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - sglang_config: - prefill: - disaggregation-mode: "prefill" - served-model-name: "deepseek-ai/DeepSeek-R1" - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.95 - max-total-tokens: 32768 - chunked-prefill-size: 24576 - cuda-graph-max-bs: 256 - max-running-requests: 2048 - scheduler-recv-interval: 1 - load-balance-method: "round_robin" - disaggregation-bootstrap-port: null - data-parallel-size: 4 - tensor-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - - decode: - disaggregation-mode: "decode" - served-model-name: "deepseek-ai/DeepSeek-R1" - prefill-round-robin-balance: true - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - disaggregation-bootstrap-port: 30001 - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.85 - chunked-prefill-size: 8192 - cuda-graph-max-bs: 1024 - max-running-requests: 2048 - scheduler-recv-interval: 1 - data-parallel-size: 8 - tensor-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: 300 - concurrencies: "2048" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_4p1d_c2048.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_4p1d_c2048.yaml deleted file mode 100644 index fc2b0f3366..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_4p1d_c2048.yaml +++ /dev/null @@ -1,119 +0,0 @@ -# DeepSeek-R1-0528 FP4 8k/1k B200 dynamo-sglang disagg — MTP, 4P:1D (c2048) -# Topology: 4 prefill workers (TP4/EP4 dp-attn) + 1 decode worker (TP8/EP8 dp-attn) = 24 GPU. -# Dedicated high-concurrency tune: -# SGLANG_HACK_SEQ_BOOTSTRAP_ROOM performance knob (not output-affecting; standard in GB200 recipes). -# max-running-requests raised to 2048 to avoid admission throttle at high concurrency. -# scheduler-recv-interval: 1 on decode. -# symm-mem + allreduce fusion retained. -# prefill stream-interval: 100; decode stream-interval: 34 (tuned for the c2048 batch cadence). -name: b200-fp4-dsr1_8k1k_mtp_4p1d_c2048 -model: - path: "dsr1" - container: "lmsysorg/sglang:v0.5.12.post1" - precision: fp4 -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 2 - prefill_workers: 4 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 -frontend: - type: dynamo -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true -backend: - prefill_environment: &env - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - DYN_REQUEST_PLANE: nats - SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - decode_environment: *env - sglang_config: - prefill: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: prefill - load-balance-method: round_robin - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 9600 - max-running-requests: 2048 - disable-cuda-graph: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - moe-dense-tp-size: 1 - enable-flashinfer-allreduce-fusion: true - enable-symm-mem: true - disable-radix-cache: true - stream-interval: 100 - watchdog-timeout: 1000000 - decode: - served-model-name: deepseek-ai/DeepSeek-R1 - trust-remote-code: true - quantization: modelopt_fp4 - disaggregation-mode: decode - prefill-round-robin-balance: true - disaggregation-transfer-backend: nixl - mem-fraction-static: 0.75 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - context-length: 9600 - max-running-requests: 2048 - cuda-graph-max-bs: 1024 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - attention-backend: trtllm_mla - kv-cache-dtype: fp8_e4m3 - moe-runner-backend: flashinfer_trtllm - moe-dense-tp-size: 1 - enable-flashinfer-allreduce-fusion: true - enable-symm-mem: true - disable-radix-cache: true - stream-interval: 34 - scheduler-recv-interval: 1 - watchdog-timeout: 1000000 - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 -benchmark: - type: sa-bench - req_rate: inf - isl: 8192 - osl: 1024 - concurrencies: '2048' - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml deleted file mode 100644 index 62bbddf84d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml +++ /dev/null @@ -1,140 +0,0 @@ -name: "b200-fp4-mtp-low-latency-1p-tp4-5d-tp8" - -# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/b200-fp4/8k1k.yaml -# base + zip_override_mtp_lowlat[0], 1p-tp4 prefill / 5d-tp8 decode). -# One flat YAML per concrete topology, matching the InferenceX glm5 disagg -# layout (sglang//-//disagg//...). - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx-sqsh - -model: - path: "dsr1" - container: "lmsysorg/sglang:v0.5.12.post1" - precision: "fp4" - -resources: - gpu_type: "b200" - prefill_nodes: 1 - decode_nodes: 5 - prefill_workers: 1 - decode_workers: 5 - gpus_per_prefill: 4 - gpus_per_node: 8 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - sglang_config: - prefill: - disaggregation-mode: "prefill" - served-model-name: "deepseek-ai/DeepSeek-R1" - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.95 - max-total-tokens: 32768 - chunked-prefill-size: 24576 - cuda-graph-max-bs: 256 - max-running-requests: 512 - scheduler-recv-interval: 10 - load-balance-method: "round_robin" - disaggregation-bootstrap-port: 30001 - data-parallel-size: 1 - tensor-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: false - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - decode: - disaggregation-mode: "decode" - served-model-name: "deepseek-ai/DeepSeek-R1" - prefill-round-robin-balance: true - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - disaggregation-bootstrap-port: 30001 - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.85 - chunked-prefill-size: 8192 - cuda-graph-max-bs: 128 - max-running-requests: 512 - scheduler-recv-interval: 10 - tensor-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: false - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: 300 - concurrencies: "4x8x16x32" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml deleted file mode 100644 index 521eddf931..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml +++ /dev/null @@ -1,140 +0,0 @@ -name: "b200-fp4-mtp-low-latency-1p-tp4-3d-tp8" - -# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/b200-fp4/8k1k.yaml -# base + zip_override_mtp_lowlat[1], 1p-tp4 prefill / 3d-tp8 decode). -# One flat YAML per concrete topology, matching the InferenceX glm5 disagg -# layout (sglang//-//disagg//...). - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx-sqsh - -model: - path: "dsr1" - container: "lmsysorg/sglang:v0.5.12.post1" - precision: "fp4" - -resources: - gpu_type: "b200" - prefill_nodes: 1 - decode_nodes: 3 - prefill_workers: 1 - decode_workers: 3 - gpus_per_prefill: 4 - gpus_per_node: 8 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - sglang_config: - prefill: - disaggregation-mode: "prefill" - served-model-name: "deepseek-ai/DeepSeek-R1" - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.95 - max-total-tokens: 32768 - chunked-prefill-size: 24576 - cuda-graph-max-bs: 256 - max-running-requests: 512 - scheduler-recv-interval: 10 - load-balance-method: "round_robin" - disaggregation-bootstrap-port: 30001 - data-parallel-size: 1 - tensor-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: false - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - decode: - disaggregation-mode: "decode" - served-model-name: "deepseek-ai/DeepSeek-R1" - prefill-round-robin-balance: true - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - disaggregation-bootstrap-port: 30001 - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.85 - chunked-prefill-size: 8192 - cuda-graph-max-bs: 128 - max-running-requests: 512 - scheduler-recv-interval: 10 - tensor-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: false - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: 300 - concurrencies: "32x64" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml deleted file mode 100644 index 9dfc6331a8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml +++ /dev/null @@ -1,140 +0,0 @@ -name: "b200-fp4-mtp-low-latency-1p-tp4-1d-tp8" - -# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/b200-fp4/8k1k.yaml -# base + zip_override_mtp_lowlat[2], 1p-tp4 prefill / 1d-tp8 decode). -# One flat YAML per concrete topology, matching the InferenceX glm5 disagg -# layout (sglang//-//disagg//...). - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx-sqsh - -model: - path: "dsr1" - container: "lmsysorg/sglang:v0.5.12.post1" - precision: "fp4" - -resources: - gpu_type: "b200" - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_node: 8 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - sglang_config: - prefill: - disaggregation-mode: "prefill" - served-model-name: "deepseek-ai/DeepSeek-R1" - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.95 - max-total-tokens: 32768 - chunked-prefill-size: 24576 - cuda-graph-max-bs: 256 - max-running-requests: 512 - scheduler-recv-interval: 10 - load-balance-method: "round_robin" - disaggregation-bootstrap-port: 30001 - data-parallel-size: 1 - tensor-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: false - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - decode: - disaggregation-mode: "decode" - served-model-name: "deepseek-ai/DeepSeek-R1" - prefill-round-robin-balance: true - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - disaggregation-bootstrap-port: 30001 - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.85 - chunked-prefill-size: 8192 - cuda-graph-max-bs: 128 - max-running-requests: 512 - scheduler-recv-interval: 10 - tensor-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: false - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: 300 - concurrencies: "32" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_3.yaml deleted file mode 100644 index ba9907840f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_3.yaml +++ /dev/null @@ -1,152 +0,0 @@ -name: gb300-fp8-glm5-mtp_1k1k_hightpt_3 - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: fp8 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 6 - prefill_workers: 6 - decode_nodes: 12 - decode_workers: 1 -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 9 -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). - # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' - - sglang_config: - prefill: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - - # Other flags - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - # moe-runner-backend: "cutedsl" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - tensor-parallel-size: 48 - expert-parallel-size: 48 - data-parallel-size: 48 - enable-dp-lm-head: true - enable-dp-attention: true - moe-dense-tp-size: 1 - ep-num-redundant-experts: 32 - ep-dispatch-algorithm: static - moe-a2a-backend: deepep - deepep-mode: low_latency - deepep-config: /configs/deepep_config.json - max-running-requests: 6144 - cuda-graph-max-bs: 128 - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 1024 - osl: 1024 - concurrencies: '6500' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_4.yaml deleted file mode 100644 index ee2b0a2026..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_4.yaml +++ /dev/null @@ -1,152 +0,0 @@ -name: gb300-fp8-glm5-mtp_1k1k_hightpt_4 - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: fp8 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 4 - prefill_workers: 4 - decode_nodes: 14 - decode_workers: 1 -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 9 -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). - # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' - - sglang_config: - prefill: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - - # Other flags - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - # moe-runner-backend: "cutedsl" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - tensor-parallel-size: 56 - expert-parallel-size: 56 - data-parallel-size: 56 - enable-dp-lm-head: true - enable-dp-attention: true - moe-dense-tp-size: 1 - ep-num-redundant-experts: 24 - ep-dispatch-algorithm: static - moe-a2a-backend: deepep - deepep-mode: low_latency - deepep-config: /configs/deepep_config.json - max-running-requests: 5600 - cuda-graph-max-bs: 100 - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 1024 - osl: 1024 - concurrencies: '5700' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml deleted file mode 100644 index cd01ba6b73..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml +++ /dev/null @@ -1,144 +0,0 @@ -name: gb300-fp8-glm5-mtp_1k1k_lowlat_0 - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: fp8 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - decode_nodes: 17 - decode_workers: 17 -frontend: - type: dynamo -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). - # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' - - sglang_config: - prefill: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - - # Other flags - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - # moe-runner-backend: "cutedsl" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - enable-flashinfer-allreduce-fusion: true - moe-runner-backend: flashinfer_trtllm - max-running-requests: 32 - cuda-graph-max-bs: 32 - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 1024 - osl: 1024 - concurrencies: 512x256x128x64 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml deleted file mode 100644 index 1ec2d053a3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml +++ /dev/null @@ -1,144 +0,0 @@ -name: gb300-fp8-glm5-mtp_1k1k_lowlat_1 - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: fp8 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - decode_nodes: 17 - decode_workers: 17 -frontend: - type: dynamo -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). - # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' - - sglang_config: - prefill: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - - # Other flags - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - # moe-runner-backend: "cutedsl" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - enable-flashinfer-allreduce-fusion: true - moe-runner-backend: flashinfer_trtllm - max-running-requests: 1 - cuda-graph-max-bs: 1 - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 1024 - osl: 1024 - concurrencies: '32' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_0.yaml deleted file mode 100644 index 6fbd1df10b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_0.yaml +++ /dev/null @@ -1,152 +0,0 @@ -name: gb300-fp8-glm5-mtp_8k1k_hightpt_0 - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: fp8 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 14 - prefill_workers: 14 - decode_nodes: 4 - decode_workers: 1 -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 9 -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). - # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' - - sglang_config: - prefill: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - - # Other flags - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - # moe-runner-backend: "cutedsl" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - tensor-parallel-size: 16 - expert-parallel-size: 16 - data-parallel-size: 16 - enable-dp-lm-head: true - enable-dp-attention: true - moe-dense-tp-size: 1 - ep-num-redundant-experts: 32 - ep-dispatch-algorithm: static - moe-a2a-backend: deepep - deepep-mode: low_latency - deepep-config: /configs/deepep_config.json - max-running-requests: 2800 - cuda-graph-max-bs: 175 - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 8192 - osl: 1024 - concurrencies: '2800' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_1.yaml deleted file mode 100644 index 3c1a641a82..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_1.yaml +++ /dev/null @@ -1,152 +0,0 @@ -name: gb300-fp8-glm5-mtp_8k1k_hightpt_1 - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: fp8 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 12 - prefill_workers: 12 - decode_nodes: 6 - decode_workers: 1 -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 9 -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). - # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' - - sglang_config: - prefill: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - - # Other flags - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - # moe-runner-backend: "cutedsl" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - tensor-parallel-size: 24 - expert-parallel-size: 24 - data-parallel-size: 24 - enable-dp-lm-head: true - enable-dp-attention: true - moe-dense-tp-size: 1 - ep-num-redundant-experts: 32 - ep-dispatch-algorithm: static - moe-a2a-backend: deepep - deepep-mode: low_latency - deepep-config: /configs/deepep_config.json - max-running-requests: 1680 - cuda-graph-max-bs: 70 - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 8192 - osl: 1024 - concurrencies: '1700' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_2.yaml deleted file mode 100644 index 1ee087dfa2..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_2.yaml +++ /dev/null @@ -1,152 +0,0 @@ -name: gb300-fp8-glm5-mtp_8k1k_hightpt_2 - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: fp8 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 10 - prefill_workers: 10 - decode_nodes: 8 - decode_workers: 1 -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 9 -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). - # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' - - sglang_config: - prefill: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - - # Other flags - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - # moe-runner-backend: "cutedsl" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - tensor-parallel-size: 32 - expert-parallel-size: 32 - data-parallel-size: 32 - enable-dp-lm-head: true - enable-dp-attention: true - moe-dense-tp-size: 1 - ep-num-redundant-experts: 32 - ep-dispatch-algorithm: static - moe-a2a-backend: deepep - deepep-mode: low_latency - deepep-config: /configs/deepep_config.json - max-running-requests: 1280 - cuda-graph-max-bs: 40 - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 8192 - osl: 1024 - concurrencies: '1300' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_3.yaml deleted file mode 100644 index a03cdef22d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_3.yaml +++ /dev/null @@ -1,152 +0,0 @@ -name: gb300-fp8-glm5-mtp_8k1k_hightpt_3 - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: fp8 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 8 - prefill_workers: 8 - decode_nodes: 10 - decode_workers: 1 -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 9 -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). - # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' - - sglang_config: - prefill: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - - # Other flags - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - # moe-runner-backend: "cutedsl" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - tensor-parallel-size: 40 - expert-parallel-size: 40 - data-parallel-size: 40 - enable-dp-lm-head: true - enable-dp-attention: true - moe-dense-tp-size: 1 - ep-num-redundant-experts: 24 - ep-dispatch-algorithm: static - moe-a2a-backend: deepep - deepep-mode: low_latency - deepep-config: /configs/deepep_config.json - max-running-requests: 880 - cuda-graph-max-bs: 22 - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 8192 - osl: 1024 - concurrencies: '900' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml deleted file mode 100644 index baf3aa3609..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml +++ /dev/null @@ -1,144 +0,0 @@ -name: gb300-fp8-glm5-mtp_8k1k_lowlat_0 - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: fp8 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - decode_nodes: 9 - decode_workers: 9 -frontend: - type: dynamo -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). - # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' - - sglang_config: - prefill: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - - # Other flags - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - # moe-runner-backend: "cutedsl" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - enable-flashinfer-allreduce-fusion: true - moe-runner-backend: flashinfer_trtllm - max-running-requests: 15 - cuda-graph-max-bs: 15 - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 8192 - osl: 1024 - concurrencies: '150' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml deleted file mode 100644 index c386534d01..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml +++ /dev/null @@ -1,144 +0,0 @@ -name: gb300-fp8-glm5-mtp_8k1k_lowlat_1 - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: fp8 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - decode_nodes: 17 - decode_workers: 17 -frontend: - type: dynamo -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). - # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' - - sglang_config: - prefill: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - - # Other flags - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - # moe-runner-backend: "cutedsl" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - enable-flashinfer-allreduce-fusion: true - moe-runner-backend: flashinfer_trtllm - max-running-requests: 8 - cuda-graph-max-bs: 8 - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 8192 - osl: 1024 - concurrencies: 128x64x32 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml deleted file mode 100644 index 561ac280cc..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml +++ /dev/null @@ -1,144 +0,0 @@ -name: gb300-fp8-glm5-mtp_8k1k_lowlat_2 - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: fp8 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - decode_nodes: 17 - decode_workers: 17 -frontend: - type: dynamo -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - # DeepEP per-rank dispatch buffer; must be >= ceil(cuda_graph_max_bs / dp_size). - # Default 128 overflows with large DP + batch (e.g. 4096/24 ~= 171 > 128). Limit 1024. - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' - - sglang_config: - prefill: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - - # Other flags - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - - # Disaggregation mode - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - # moe-runner-backend: "cutedsl" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - enable-flashinfer-allreduce-fusion: true - moe-runner-backend: flashinfer_trtllm - max-running-requests: 1 - cuda-graph-max-bs: 1 - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 8192 - osl: 1024 - concurrencies: '24' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml deleted file mode 100644 index c44f69ef39..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml +++ /dev/null @@ -1,167 +0,0 @@ -name: "disagg-h200-1p1d-pcp8-tp8-dp8-mtp6-hicache" - -# 1P1D GLM-5.2 FP8 AgentX topology on H200 with EAGLE MTP and prefill-side -# hierarchical KV cache. The decode-side KV cache remains GPU-resident. - -model: - path: "hf:zai-org/GLM-5.2-FP8" - container: "lmsysorg/sglang:v0.5.16-cu130" - precision: "fp8" - -dynamo: - install: true - wheel: "1.3.0.dev1" - -resources: - gpu_type: h200 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Correlation-ID - env: - DYN_ROUTER_TEMPERATURE: "10000000" - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-reset-states: true - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - prefill_environment: - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache" - PIP_BREAK_SYSTEM_PACKAGES: "1" - PYTHONUNBUFFERED: "1" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_OPT_USE_TOPK_V2: "1" - SGLANG_TIMEOUT_KEEP_ALIVE: "900" - decode_environment: - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache" - PIP_BREAK_SYSTEM_PACKAGES: "1" - PYTHONUNBUFFERED: "1" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_OPT_USE_TOPK_V2: "1" - SGLANG_TIMEOUT_KEEP_ALIVE: "900" - SGLANG_SIMULATE_ACC_LEN: "3.78" - SGLANG_SIMULATE_ACC_METHOD: "match-expected" - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - - sglang_config: - prefill: - host: 0.0.0.0 - model-path: /model/ - served-model-name: zai-org/GLM-5.2-FP8 - trust-remote-code: true - tool-call-parser: glm47 - reasoning-parser: glm45 - tp-size: 8 - ep-size: 1 - attn-cp-size: 8 - enable-prefill-cp: true - cp-strategy: interleave - enable-dsa-cache-layer-split: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - kv-cache-dtype: fp8_e4m3 - context-length: 1048576 - max-total-tokens: 1048576 - chunked-prefill-size: 32768 - mem-fraction-static: 0.85 - max-running-requests: 32 - page-size: 64 - enable-hierarchical-cache: true - hicache-size: 64 - hicache-mem-layout: layer_first - hicache-io-backend: kernel - hicache-write-policy: write_back - speculative-algorithm: EAGLE - speculative-num-steps: 6 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - watchdog-timeout: 86400 - stream-interval: 60 - enable-metrics: true - enable-cache-report: true - - decode: - host: 0.0.0.0 - model-path: /model/ - served-model-name: zai-org/GLM-5.2-FP8 - trust-remote-code: true - tool-call-parser: glm47 - reasoning-parser: glm45 - tp-size: 8 - dp-size: 8 - ep-size: 1 - enable-dp-attention: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - kv-cache-dtype: fp8_e4m3 - dsa-decode-backend: flashmla_kv - context-length: 1048576 - max-total-tokens: 1048576 - mem-fraction-static: 0.85 - max-running-requests: 200 - page-size: 64 - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 6 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - watchdog-timeout: 86400 - stream-interval: 60 - enable-metrics: true - enable-cache-report: true - -sbatch_directives: - mem: "0" - -srun_options: - mem: "0" - container-remap-root: "" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml deleted file mode 100644 index 4d5a3696a5..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml +++ /dev/null @@ -1,166 +0,0 @@ -name: "disagg-h200-2p2d-pcp8-tp8-dp8-mtp" - -# Refresh of the 2P2D topology from Actions run 30133535261 with EAGLE MTP. -# The released SGLang image does not combine MTP with HiSparse, so decode KV -# cache remains GPU-resident. MTP uses the committed GLM-5.2 golden thinking-on -# acceptance length for three speculative steps. - -model: - path: "hf:zai-org/GLM-5.2-FP8" - container: "lmsysorg/sglang:v0.5.16-cu130" - precision: "fp8" - -dynamo: - install: true - wheel: "1.3.0.dev1" - -slurm: - time_limit: "8:00:00" - -resources: - gpu_type: h200 - gpus_per_node: 8 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 2 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 32 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Correlation-ID - env: - DYN_ROUTER_TEMPERATURE: "10000000" - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: "kv" - router-reset-states: true - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - prefill_environment: - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache" - PIP_BREAK_SYSTEM_PACKAGES: "1" - PYTHONUNBUFFERED: "1" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_OPT_USE_TOPK_V2: "1" - SGLANG_TIMEOUT_KEEP_ALIVE: "900" - decode_environment: - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache" - PIP_BREAK_SYSTEM_PACKAGES: "1" - PYTHONUNBUFFERED: "1" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_OPT_USE_TOPK_V2: "1" - SGLANG_TIMEOUT_KEEP_ALIVE: "900" - SGLANG_SIMULATE_ACC_LEN: "2.99" - SGLANG_SIMULATE_ACC_METHOD: "match-expected" - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - - sglang_config: - prefill: - host: 0.0.0.0 - model-path: /model/ - served-model-name: zai-org/GLM-5.2-FP8 - trust-remote-code: true - tool-call-parser: glm47 - reasoning-parser: glm45 - tp-size: 8 - ep-size: 1 - attn-cp-size: 8 - enable-prefill-cp: true - cp-strategy: interleave - enable-dsa-cache-layer-split: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: prefill - kv-cache-dtype: fp8_e4m3 - context-length: 1048576 - max-total-tokens: 1048576 - chunked-prefill-size: 32768 - mem-fraction-static: 0.85 - max-running-requests: 32 - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - watchdog-timeout: 86400 - stream-interval: 60 - enable-metrics: true - enable-cache-report: true - - decode: - host: 0.0.0.0 - model-path: /model/ - served-model-name: zai-org/GLM-5.2-FP8 - trust-remote-code: true - tool-call-parser: glm47 - reasoning-parser: glm45 - tp-size: 8 - dp-size: 8 - ep-size: 1 - enable-dp-attention: true - disaggregation-transfer-backend: mooncake - disaggregation-mode: decode - kv-cache-dtype: fp8_e4m3 - dsa-decode-backend: flashmla_kv - context-length: 1048576 - max-total-tokens: 1048576 - mem-fraction-static: 0.85 - max-running-requests: 200 - page-size: 64 - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - watchdog-timeout: 86400 - stream-interval: 60 - enable-metrics: true - enable-cache-report: true - -sbatch_directives: - mem: "0" - -srun_options: - mem: "0" - container-remap-root: "" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c1-mtp.yaml deleted file mode 100644 index c21f566e92..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c1-mtp.yaml +++ /dev/null @@ -1,118 +0,0 @@ -# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 1. -# -# One TP8 worker serves prefill and decode on a single node. The sweep matrix -# supplies the concurrency list; agentic_srt.sh replays every point against -# this one server. Acceptance is pinned to the golden thinking-on AL for three -# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). -name: agg-b200-tp8-c1-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 - frameworks: - dynamo: 1.5.0.dev20260909 - sglang: 0.0.0.dev1+g008403017 -resources: - gpu_type: b200 - gpus_per_node: 8 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 8 -dynamo: - install: false -backend: - type: sglang - sglang_config: - aggregated: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 2 - cuda-graph-max-bs-decode: 2 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 110 - hicache-io-backend: direct - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true - aggregated_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_SIMULATE_ACC_LEN: '2.99' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - PIP_BREAK_SYSTEM_PACKAGES: '1' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -health_check: - max_attempts: 1440 - interval_seconds: 10 -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 64 -frontend: - type: dynamo - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-session-affinity-ttl-secs: 3600 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c4-mtp.yaml deleted file mode 100644 index 10c00aea48..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c4-mtp.yaml +++ /dev/null @@ -1,118 +0,0 @@ -# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 4. -# -# One TP8 worker serves prefill and decode on a single node. The sweep matrix -# supplies the concurrency list; agentic_srt.sh replays every point against -# this one server. Acceptance is pinned to the golden thinking-on AL for three -# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). -name: agg-b200-tp8-c4-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 - frameworks: - dynamo: 1.5.0.dev20260909 - sglang: 0.0.0.dev1+g008403017 -resources: - gpu_type: b200 - gpus_per_node: 8 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 8 -dynamo: - install: false -backend: - type: sglang - sglang_config: - aggregated: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 8 - cuda-graph-max-bs-decode: 8 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 110 - hicache-io-backend: direct - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true - aggregated_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_SIMULATE_ACC_LEN: '2.99' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - PIP_BREAK_SYSTEM_PACKAGES: '1' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -health_check: - max_attempts: 1440 - interval_seconds: 10 -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 64 -frontend: - type: dynamo - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-session-affinity-ttl-secs: 3600 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c8-mtp.yaml deleted file mode 100644 index aa25a5e958..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c8-mtp.yaml +++ /dev/null @@ -1,118 +0,0 @@ -# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on B200 at concurrency 8. -# -# One TP8 worker serves prefill and decode on a single node. The sweep matrix -# supplies the concurrency list; agentic_srt.sh replays every point against -# this one server. Acceptance is pinned to the golden thinking-on AL for three -# speculative tokens (golden_al_distribution/glm5.2_mtp.yaml). -name: agg-b200-tp8-c8-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 - frameworks: - dynamo: 1.5.0.dev20260909 - sglang: 0.0.0.dev1+g008403017 -resources: - gpu_type: b200 - gpus_per_node: 8 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 8 -dynamo: - install: false -backend: - type: sglang - sglang_config: - aggregated: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 110 - hicache-io-backend: direct - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true - aggregated_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_SIMULATE_ACC_LEN: '2.99' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - PIP_BREAK_SYSTEM_PACKAGES: '1' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -health_check: - max_attempts: 1440 - interval_seconds: 10 -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 64 -frontend: - type: dynamo - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-session-affinity-ttl-secs: 3600 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p1d-dep8-dep8-c64-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p1d-dep8-dep8-c64-mtp.yaml deleted file mode 100644 index a121cc3573..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p1d-dep8-dep8-c64-mtp.yaml +++ /dev/null @@ -1,193 +0,0 @@ -name: disagg-b200-1p1d-dep8-dep8-c64-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 - frameworks: - dynamo: 1.5.0.dev20260909 - sglang: 0.0.0.dev1+g008403017 -resources: - gpu_type: b200 - gpus_per_node: 8 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 - gpus_per_prefill: 8 - prefill_nodes: 1 - prefill_workers: 1 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: false -backend: - type: sglang - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - SGLANG_SIMULATE_ACC_LEN: '2.5' - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '1800' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - SGLANG_SIMULATE_ACC_LEN: '2.5' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - sglang_config: - prefill: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - chunked-prefill-size: 65536 - max-prefill-tokens: 8192 - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutlass - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 110 - hicache-io-backend: direct - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true - decode: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 128 - cuda-graph-max-bs-decode: 128 - chunked-prefill-size: 64 - context-length: 1048576 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutedsl - fp4-gemm-backend: flashinfer_cutlass - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.85 - disaggregation-decode-extra-slots: 0 - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true - deepep-config: /configs/deepep_config.json - deepep-mode: low_latency - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - speculative-moe-a2a-backend: deepep - speculative-moe-runner-backend: deep_gemm -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 64 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p4d-dep8-tp4-c48-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p4d-dep8-tp4-c48-mtp.yaml deleted file mode 100644 index 084b6b791e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p4d-dep8-tp4-c48-mtp.yaml +++ /dev/null @@ -1,185 +0,0 @@ -name: disagg-b200-1p4d-dep8-tp4-c48-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-20260910-00840301 - frameworks: - dynamo: 1.5.0.dev20260909 - sglang: 0.0.0.dev1+g008403017 -resources: - gpu_type: b200 - gpus_per_node: 8 - decode_nodes: 2 - decode_workers: 4 - gpus_per_decode: 4 - gpus_per_prefill: 8 - prefill_nodes: 1 - prefill_workers: 1 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - install: false -backend: - type: sglang - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '3600' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - SGLANG_SIMULATE_ACC_LEN: '2.5' - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT: '3600' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_DEFAULT_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - SGLANG_SIMULATE_ACC_LEN: '2.5' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - sglang_config: - prefill: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutlass - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 110 - hicache-io-backend: direct - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true - decode: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 16 - cuda-graph-max-bs-decode: 16 - chunked-prefill-size: 64 - context-length: 1048576 - dsa-prefill-backend: trtllm - dsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_cutlass - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.9 - disaggregation-decode-extra-slots: 0 - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 64 -sbatch_directives: - mem: '0' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/agg-gb200-tp8-c2-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/agg-gb200-tp8-c2-mtp.yaml deleted file mode 100644 index 1b0f392a33..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/agg-gb200-tp8-c2-mtp.yaml +++ /dev/null @@ -1,122 +0,0 @@ -name: agg-gb200-tp8-c2-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 - gpus_per_agg: 8 - agg_nodes: 2 - agg_workers: 1 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - install: true -backend: - type: sglang - sglang_config: - aggregated: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 10 - cuda-graph-max-bs: 10 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - enable-metrics: true - aggregated_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_SIMULATE_ACC_LEN: '3.33' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - PIP_BREAK_SYSTEM_PACKAGES: '1' -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/agg-gb200-tp8-c4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/agg-gb200-tp8-c4-mtp.yaml deleted file mode 100644 index fc6a79bcaf..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/agg-gb200-tp8-c4-mtp.yaml +++ /dev/null @@ -1,122 +0,0 @@ -name: agg-gb200-tp8-c4-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 - gpus_per_agg: 8 - agg_nodes: 2 - agg_workers: 1 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - install: true -backend: - type: sglang - sglang_config: - aggregated: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 10 - cuda-graph-max-bs: 10 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - enable-metrics: true - aggregated_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_SIMULATE_ACC_LEN: '3.33' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - PIP_BREAK_SYSTEM_PACKAGES: '1' -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/agg-gb200-tp8-c8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/agg-gb200-tp8-c8-mtp.yaml deleted file mode 100644 index 4fc4b18651..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/agg-gb200-tp8-c8-mtp.yaml +++ /dev/null @@ -1,122 +0,0 @@ -name: agg-gb200-tp8-c8-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 - gpus_per_agg: 8 - agg_nodes: 2 - agg_workers: 1 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - install: true -backend: - type: sglang - sglang_config: - aggregated: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 10 - cuda-graph-max-bs: 10 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - enable-metrics: true - aggregated_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_SIMULATE_ACC_LEN: '3.33' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - PIP_BREAK_SYSTEM_PACKAGES: '1' -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/disagg-gb200-1p4d-dep8-tp4-c48-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/disagg-gb200-1p4d-dep8-tp4-c48-mtp.yaml deleted file mode 100644 index e779088aa2..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/disagg-gb200-1p4d-dep8-tp4-c48-mtp.yaml +++ /dev/null @@ -1,193 +0,0 @@ -name: disagg-gb200-1p4d-dep8-tp4-c48-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 - decode_nodes: 4 - decode_workers: 4 - gpus_per_decode: 4 - gpus_per_prefill: 8 - prefill_nodes: 2 - prefill_workers: 1 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - install: true -backend: - type: sglang - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - SGLANG_SIMULATE_ACC_LEN: '2.5' - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - SGLANG_SIMULATE_ACC_LEN: '2.5' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - sglang_config: - prefill: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - max-running-requests: 16 - cuda-graph-max-bs: 16 - disable-cuda-graph: true - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutlass - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - speculative-algorithm: EAGLE - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 2 - enable-metrics: true - enable-cache-report: true - decode: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 16 - cuda-graph-max-bs: 16 - chunked-prefill-size: 64 - context-length: 1048576 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_cutlass - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.9 - disaggregation-decode-extra-slots: 0 - enable-metrics: true - enable-cache-report: true -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/disagg-gb200-1p6d-dep8-tp4-c45-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/disagg-gb200-1p6d-dep8-tp4-c45-mtp.yaml deleted file mode 100644 index 7e305253ad..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/disagg-gb200-1p6d-dep8-tp4-c45-mtp.yaml +++ /dev/null @@ -1,193 +0,0 @@ -name: disagg-gb200-1p6d-dep8-tp4-c45-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 - decode_nodes: 6 - decode_workers: 6 - gpus_per_decode: 4 - gpus_per_prefill: 8 - prefill_nodes: 2 - prefill_workers: 1 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - install: true -backend: - type: sglang - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - SGLANG_SIMULATE_ACC_LEN: '2.5' - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - SGLANG_SIMULATE_ACC_LEN: '2.5' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - sglang_config: - prefill: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - max-running-requests: 16 - cuda-graph-max-bs: 16 - disable-cuda-graph: true - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutlass - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - speculative-algorithm: EAGLE - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 2 - enable-metrics: true - enable-cache-report: true - decode: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 16 - cuda-graph-max-bs: 16 - chunked-prefill-size: 64 - context-length: 1048576 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_cutlass - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.9 - disaggregation-decode-extra-slots: 0 - enable-metrics: true - enable-cache-report: true -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/disagg-gb200-2p1d-dep8-dep16-c128-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/disagg-gb200-2p1d-dep8-dep16-c128-mtp.yaml deleted file mode 100644 index 76dd1d5794..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/disagg-gb200-2p1d-dep8-dep16-c128-mtp.yaml +++ /dev/null @@ -1,201 +0,0 @@ -name: disagg-gb200-2p1d-dep8-dep16-c128-mtp -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: nightly-dev-cu13-20260805-211ee642 -resources: - gpu_type: gb200 - gpus_per_node: 4 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - gpus_per_prefill: 8 - prefill_nodes: 4 - prefill_workers: 2 -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None -dynamo: - hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - install: true -backend: - type: sglang - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - SGLANG_SIMULATE_ACC_LEN: '2.5' - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: cuda_copy,cuda_ipc,sm,self,tcp - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - SGLANG_SIMULATE_ACC_LEN: '2.5' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - sglang_config: - prefill: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - chunked-prefill-size: 65536 - max-prefill-tokens: 8192 - max-running-requests: 16 - cuda-graph-max-bs: 16 - disable-cuda-graph: true - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutlass - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 100 - hicache-io-backend: direct - speculative-algorithm: EAGLE - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 2 - enable-metrics: true - enable-cache-report: true - decode: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 144 - cuda-graph-max-bs: 144 - chunked-prefill-size: 64 - context-length: 1048576 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutedsl - fp4-gemm-backend: flashinfer_cutlass - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.85 - disaggregation-decode-extra-slots: 0 - enable-metrics: true - enable-cache-report: true - deepep-config: /configs/deepep_config.json - deepep-mode: low_latency - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - speculative-moe-a2a-backend: deepep - speculative-moe-runner-backend: deep_gemm -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/glm5.2-agentx-agg.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/glm5.2-agentx-agg.yaml deleted file mode 100644 index f82b3a2581..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/glm5.2-agentx-agg.yaml +++ /dev/null @@ -1,140 +0,0 @@ -name: gb200-fp4-glm5.2-agentx-agg -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:v0.5.17-cu130 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: 0.5.17 -resources: - gpu_type: gb200 - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 -frontend: - type: dynamo - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-session-affinity-ttl-secs: 3600 -dynamo: - hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - install: true -backend: - type: sglang - sglang_config: - aggregated: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 10 - cuda-graph-max-bs: 10 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - bf16-gemm-backend: cutedsl - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.83 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true - aggregated_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_TIMEOUT_KEEP_ALIVE: '900' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_SIMULATE_ACC_LEN: '2.99' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - PIP_BREAK_SYSTEM_PACKAGES: '1' - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self - NVSHMEM_REMOTE_TRANSPORT: none -health_check: - max_attempts: 1440 - interval_seconds: 10 -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 -benchmark: - type: custom - concurrencies: [1] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/glm5.2-agentx.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/glm5.2-agentx.yaml deleted file mode 100644 index 41bf58cd06..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic/glm5.2-agentx.yaml +++ /dev/null @@ -1,217 +0,0 @@ -base: - name: gb200-fp4-glm5.2-agentx - model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 - identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - revision: aec724e8c7b8ee9db3b48c01c320f63f9cdaf8aa - container: - image: lmsysorg/sglang:v0.5.17-cu130 - frameworks: - dynamo: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - sglang: 0.5.17 - resources: - gpu_type: gb200 - gpus_per_node: 4 - frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-kv-events: true - router-temperature: 0 - kv-cache-block-size: 64 - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-session-affinity-ttl-secs: 3600 - dynamo: - hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - install: true - backend: - type: sglang - kv_events_config: true - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_TIMEOUT_KEEP_ALIVE: '900' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '2048' - SGLANG_SIMULATE_ACC_LEN: '2.99' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_TIMEOUT_KEEP_ALIVE: '900' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '2048' - SGLANG_SIMULATE_ACC_LEN: '2.99' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - sglang_config: - prefill: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 4 - attn-cp-size: 1 - enable-prefill-cp: false - enable-dsa-cache-layer-split: false - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - load-balance-method: total_tokens - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - max-total-tokens: 1048576 - max-running-requests: 64 - cuda-graph-max-bs: 64 - disable-cuda-graph: true - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - bf16-gemm-backend: cutedsl - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.83 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true - decode: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 64 - cuda-graph-max-bs: 64 - chunked-prefill-size: 64 - context-length: 1048576 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - bf16-gemm-backend: cutedsl - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.9 - disaggregation-decode-extra-slots: 0 - watchdog-timeout: 1800 - enable-metrics: true - enable-cache-report: true - health_check: - max_attempts: 1440 - interval_seconds: 10 - benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 64 - sbatch_directives: - mem: '0' - cpus-per-task: '144' - srun_options: - mem: '0' - container-remap-root: '' - -zip_override_mtp_agentx_frontier: - name: [agentx-1p1d-tp4-hicache] - resources: - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 4 - gpus_per_prefill: 4 - prefill_nodes: 1 - prefill_workers: 1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx-agg.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx-agg.yaml deleted file mode 100644 index 87979921fc..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx-agg.yaml +++ /dev/null @@ -1,118 +0,0 @@ -# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on GB300 -# (single aggregated worker, TP8, EAGLE MTP + hierarchical-cache KV). -# -# Flat single-variant schema the agentic CI flow expects: applied via -# CONFIG_FILE= with no zip_override selector, so it must NOT be -# base:-wrapped. Concurrency is not a recipe field; the GHA matrix fans out -# one job per concurrency from the master-config conc-list into agentic_srt.sh. -name: gb300-fp4-glm5.2-agentx-agg -model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 -resources: - gpu_type: gb300 - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 -frontend: - type: dynamo - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-session-affinity-ttl-secs: 3600 -dynamo: - hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - install: true -backend: - type: sglang - sglang_config: - aggregated: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 10 - cuda-graph-max-bs: 10 - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - context-length: 1048576 - speculative-algorithm: EAGLE - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_trtllm - disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - enable-metrics: true - enable-cache-report: true - aggregated_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_REASONING_EFFORT: max - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_SIMULATE_ACC_LEN: '3.33' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - PIP_BREAK_SYSTEM_PACKAGES: '1' - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self - NVSHMEM_REMOTE_TRANSPORT: none -health_check: - max_attempts: 1440 - interval_seconds: 10 -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 64 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx.yaml deleted file mode 100644 index b04ea1752b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx.yaml +++ /dev/null @@ -1,219 +0,0 @@ -base: - name: gb300-fp4-glm5.2-agentx - model: - path: glm-5.2-fp4 - container: dynamo-sglang - precision: fp4 - resources: - gpu_type: gb300 - gpus_per_node: 4 - frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-session-affinity-ttl-secs: 3600 - dynamo: - hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd - install: true - backend: - type: sglang - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' - SGLANG_HICACHE_DEBUG_LOG: '1' - SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - SGLANG_SIMULATE_ACC_LEN: '2.5' - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_ENABLE_THINKING: '1' - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' - SGLANG_REASONING_EFFORT: max - PIP_BREAK_SYSTEM_PACKAGES: '1' - UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_DG_CACHE_DIR: /deepgemm_cache - FLASHINFER_WORKSPACE_BASE: /flashinfer_cache - SGLANG_SIMULATE_ACC_LEN: '2.5' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - sglang_config: - prefill: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - max-running-requests: 16 - cuda-graph-max-bs: 16 - disable-cuda-graph: true - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_cutlass - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.8 - context-length: 1048576 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-size: 135 - hicache-io-backend: direct - speculative-algorithm: EAGLE - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 2 - enable-metrics: true - enable-cache-report: true - decode: - served-model-name: nvidia/GLM-5.2-NVFP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - disable-radix-cache: true - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - max-running-requests: 16 - cuda-graph-max-bs: 16 - chunked-prefill-size: 64 - context-length: 1048576 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_cutlass - skip-tokenizer-init: true - stream-interval: 30 - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - mem-fraction-static: 0.9 - disaggregation-decode-extra-slots: 0 - enable-metrics: true - enable-cache-report: true - health_check: - max_attempts: 1440 - interval_seconds: 10 - benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 64 - sbatch_directives: - mem: '0' - cpus-per-task: '144' - srun_options: - mem: '0' - container-remap-root: '' -# ################# agentx ################# -zip_override_mtp_agentx_hightpt: - name: [agentx-2p1d_dep16-c128, agentx-2p1d_dep16-c192] - backend: - sglang_config: - decode: - cuda-graph-max-bs: 144 - data-parallel-size: 16 - deepep-config: /configs/deepep_config.json - deepep-mode: low_latency - enable-dp-attention: true - enable-dp-lm-head: true - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - expert-parallel-size: 16 - max-running-requests: 144 - mem-fraction-static: 0.85 - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - moe-runner-backend: flashinfer_cutedsl - speculative-moe-a2a-backend: deepep - speculative-moe-runner-backend: deep_gemm - tensor-parallel-size: 16 - prefill: - data-parallel-size: 8 - expert-parallel-size: 8 - max-prefill-tokens: 8192 - tensor-parallel-size: 8 - resources: - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - gpus_per_prefill: 8 - prefill_nodes: 4 - prefill_workers: 2 -zip_override_mtp_agentx_lowlat: - name: [agentx-1p2d_tp4-c48, agentx-1p4d_tp4-c48, agentx-1p6d_tp4-c45] - resources: - decode_nodes: [2, 4, 6] - decode_workers: [2, 4, 6] - gpus_per_decode: 4 - gpus_per_prefill: 4 - prefill_nodes: 1 - prefill_workers: 1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_0.yaml deleted file mode 100644 index 76edd78e30..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_0.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: "gb200-fp4-glm5_1k1k_lowlat_0" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 2112 - mem-fraction-static: 0.785 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 2112 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - ep-num-redundant-experts: 0 - ep-dispatch-algorithm: "static" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - deepep-config: "/configs/deepep_config.json" - moe-runner-backend: "flashinfer_cutedsl" - enable-flashinfer-allreduce-fusion: false - mem-fraction-static: 0.866 - max-running-requests: 2023 - cuda-graph-max-bs: 2023 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "2115" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_1.yaml deleted file mode 100644 index 7ff2fb9e63..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_1.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: "gb200-fp4-glm5_1k1k_lowlat_1" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 8 - decode_workers: 1 - gpus_per_decode: 32 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 2112 - mem-fraction-static: 0.785 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 2112 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 32 - data-parallel-size: 32 - expert-parallel-size: 32 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - ep-num-redundant-experts: 0 - ep-dispatch-algorithm: "static" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - deepep-config: "/configs/deepep_config.json" - moe-runner-backend: "flashinfer_cutedsl" - enable-flashinfer-allreduce-fusion: false - mem-fraction-static: 0.858 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1156" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_2.yaml deleted file mode 100644 index dde20d03d5..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_2.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: "gb200-fp4-glm5_1k1k_lowlat_2" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 8 - decode_workers: 1 - gpus_per_decode: 32 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 2112 - mem-fraction-static: 0.785 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 2112 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 32 - data-parallel-size: 32 - expert-parallel-size: 32 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - ep-num-redundant-experts: 0 - ep-dispatch-algorithm: "static" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - deepep-config: "/configs/deepep_config.json" - moe-runner-backend: "flashinfer_cutedsl" - enable-flashinfer-allreduce-fusion: false - mem-fraction-static: 0.858 - max-running-requests: 512 - cuda-graph-max-bs: 512 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "556" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_3.yaml deleted file mode 100644 index 1a084f3876..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_3.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "gb200-fp4-glm5_1k1k_lowlat_3" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 16 - decode_workers: 16 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 2112 - mem-fraction-static: 0.785 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 2112 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - load-balance-method: "round_robin" - moe-runner-backend: "flashinfer_trtllm" - enable-flashinfer-allreduce-fusion: true - mem-fraction-static: 0.918 - max-running-requests: 1 - cuda-graph-max-bs: 1 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "23" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_4.yaml deleted file mode 100644 index b8cc503e54..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_4.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "gb200-fp4-glm5_1k1k_lowlat_4" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 16 - decode_workers: 16 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 2112 - mem-fraction-static: 0.785 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 2112 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - load-balance-method: "round_robin" - moe-runner-backend: "flashinfer_trtllm" - enable-flashinfer-allreduce-fusion: true - mem-fraction-static: 0.918 - max-running-requests: 16 - cuda-graph-max-bs: 16 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "290" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_5.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_5.yaml deleted file mode 100644 index ddc60e9504..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_5.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "gb200-fp4-glm5_1k1k_lowlat_5" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 16 - decode_workers: 16 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 2112 - mem-fraction-static: 0.785 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 2112 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - load-balance-method: "round_robin" - moe-runner-backend: "flashinfer_trtllm" - enable-flashinfer-allreduce-fusion: true - mem-fraction-static: 0.918 - max-running-requests: 4 - cuda-graph-max-bs: 4 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "73" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_maxtpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_maxtpt_0.yaml deleted file mode 100644 index 3f7a83b6c3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_maxtpt_0.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: "gb200-fp4-glm5_1k1k_maxtpt_0" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 2 - gpus_per_prefill: 4 - decode_nodes: 8 - decode_workers: 1 - gpus_per_decode: 32 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 2112 - mem-fraction-static: 0.812 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 2112 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 32 - data-parallel-size: 32 - expert-parallel-size: 32 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - ep-num-redundant-experts: 0 - ep-dispatch-algorithm: "static" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - deepep-config: "/configs/deepep_config.json" - moe-runner-backend: "flashinfer_cutedsl" - enable-flashinfer-allreduce-fusion: false - mem-fraction-static: 0.858 - max-running-requests: 2048 - cuda-graph-max-bs: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "3160" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml deleted file mode 100644 index eb5b2467e8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "gb200-fp4-glm5_8k1k_lowlat_0" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 4 - decode_workers: 4 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 72 - cuda-graph-max-bs: 72 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 9280 - mem-fraction-static: 0.8 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 9600 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - load-balance-method: "round_robin" - moe-runner-backend: "flashinfer_trtllm" - enable-flashinfer-allreduce-fusion: true - mem-fraction-static: 0.94 - max-running-requests: 32 - cuda-graph-max-bs: 32 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "133" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml deleted file mode 100644 index 07b90a7261..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "gb200-fp4-glm5_8k1k_lowlat_1" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 6 - decode_workers: 6 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 72 - cuda-graph-max-bs: 72 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 9280 - mem-fraction-static: 0.8 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 9600 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - load-balance-method: "round_robin" - moe-runner-backend: "flashinfer_trtllm" - enable-flashinfer-allreduce-fusion: true - mem-fraction-static: 0.94 - max-running-requests: 22 - cuda-graph-max-bs: 22 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "146" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml deleted file mode 100644 index f7faa73fe8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "gb200-fp4-glm5_8k1k_lowlat_2" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 6 - decode_workers: 6 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 72 - cuda-graph-max-bs: 72 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 9280 - mem-fraction-static: 0.8 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 9600 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - load-balance-method: "round_robin" - moe-runner-backend: "flashinfer_trtllm" - enable-flashinfer-allreduce-fusion: true - mem-fraction-static: 0.94 - max-running-requests: 16 - cuda-graph-max-bs: 16 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "103" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml deleted file mode 100644 index 242a637dfa..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "gb200-fp4-glm5_8k1k_lowlat_3" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 8 - decode_workers: 8 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 72 - cuda-graph-max-bs: 72 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 9280 - mem-fraction-static: 0.8 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 9600 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - load-balance-method: "round_robin" - moe-runner-backend: "flashinfer_trtllm" - enable-flashinfer-allreduce-fusion: true - mem-fraction-static: 0.94 - max-running-requests: 15 - cuda-graph-max-bs: 15 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "130" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml deleted file mode 100644 index 59a3d9fc28..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "gb200-fp4-glm5_8k1k_lowlat_4" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 16 - decode_workers: 16 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 72 - cuda-graph-max-bs: 72 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 9280 - mem-fraction-static: 0.785 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 9600 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - load-balance-method: "round_robin" - moe-runner-backend: "flashinfer_trtllm" - enable-flashinfer-allreduce-fusion: true - mem-fraction-static: 0.919 - max-running-requests: 6 - cuda-graph-max-bs: 6 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "113" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_5.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_5.yaml deleted file mode 100644 index 595c310d30..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_5.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "gb200-fp4-glm5_8k1k_lowlat_5" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 16 - decode_workers: 16 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 72 - cuda-graph-max-bs: 72 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 9280 - mem-fraction-static: 0.785 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 9600 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - load-balance-method: "round_robin" - moe-runner-backend: "flashinfer_trtllm" - enable-flashinfer-allreduce-fusion: true - mem-fraction-static: 0.919 - max-running-requests: 1 - cuda-graph-max-bs: 1 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "23" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml deleted file mode 100644 index 8e383ee108..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: "gb200-fp4-glm5_8k1k_maxtpt_0" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 4 - prefill_workers: 4 - gpus_per_prefill: 4 - decode_nodes: 8 - decode_workers: 1 - gpus_per_decode: 32 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 72 - cuda-graph-max-bs: 72 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 9280 - mem-fraction-static: 0.812 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 9600 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 32 - data-parallel-size: 32 - expert-parallel-size: 32 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - ep-num-redundant-experts: 0 - ep-dispatch-algorithm: "static" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - deepep-config: "/configs/deepep_config.json" - moe-runner-backend: "flashinfer_cutedsl" - enable-flashinfer-allreduce-fusion: false - mem-fraction-static: 0.812 - max-running-requests: 599 - cuda-graph-max-bs: 599 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "989" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml deleted file mode 100644 index da9815cb24..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: "gb200-fp4-glm5_8k1k_maxtpt_1" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 4 - prefill_workers: 4 - gpus_per_prefill: 4 - decode_nodes: 8 - decode_workers: 1 - gpus_per_decode: 32 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 72 - cuda-graph-max-bs: 72 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 9280 - mem-fraction-static: 0.812 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 9600 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 32 - data-parallel-size: 32 - expert-parallel-size: 32 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - ep-num-redundant-experts: 0 - ep-dispatch-algorithm: "static" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - deepep-config: "/configs/deepep_config.json" - moe-runner-backend: "flashinfer_cutedsl" - enable-flashinfer-allreduce-fusion: false - mem-fraction-static: 0.812 - max-running-requests: 512 - cuda-graph-max-bs: 512 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "686" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml deleted file mode 100644 index f5b158baee..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: "gb200-fp4-glm5_8k1k_maxtpt_2" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 6 - prefill_workers: 6 - gpus_per_prefill: 4 - decode_nodes: 8 - decode_workers: 1 - gpus_per_decode: 32 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 72 - cuda-graph-max-bs: 72 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 9280 - mem-fraction-static: 0.812 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 9600 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 32 - data-parallel-size: 32 - expert-parallel-size: 32 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - ep-num-redundant-experts: 0 - ep-dispatch-algorithm: "static" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - deepep-config: "/configs/deepep_config.json" - moe-runner-backend: "flashinfer_cutedsl" - enable-flashinfer-allreduce-fusion: false - mem-fraction-static: 0.812 - max-running-requests: 914 - cuda-graph-max-bs: 914 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1497" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_3.yaml deleted file mode 100644 index 35ae99e403..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_3.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: "gb200-fp4-glm5_8k1k_maxtpt_3" - -# Ported from upstream srt-slurm recipes/gb200-fp4/glm5.yaml (PR #211). -# Upstream uses a single combined file with `zip_override_*` arrays expanded -# by srtctl across zip indices. We split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention (see ../../gb300-fp4/). -# All shared base envs and the prefill sglang_config are inlined here verbatim -# from the upstream `base:` block; the decode block is the upstream base plus -# the topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -dynamo: - version: "1.2.1" - -slurm: - time_limit: "03:00:00" - -# Mirror the gb300-fp4 glm5 recipes: cpus-per-task=144 avoids the 1-CPU default -# that turns dynamo install + sglang weight load into a serial crawl; mem=0 -# grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 10 - prefill_workers: 10 - gpus_per_prefill: 4 - decode_nodes: 8 - decode_workers: 1 - gpus_per_decode: 32 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - max-running-requests: 72 - cuda-graph-max-bs: 72 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - context-length: 9280 - mem-fraction-static: 0.812 - nsa-prefill-backend: "trtllm" - nsa-decode-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - context-length: 9600 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - # Parallelism (override from upstream zip_override_*) - tensor-parallel-size: 32 - data-parallel-size: 32 - expert-parallel-size: 32 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - ep-num-redundant-experts: 0 - ep-dispatch-algorithm: "static" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - deepep-config: "/configs/deepep_config.json" - moe-runner-backend: "flashinfer_cutedsl" - enable-flashinfer-allreduce-fusion: false - mem-fraction-static: 0.812 - max-running-requests: 1712 - cuda-graph-max-bs: 1712 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "2674" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml deleted file mode 100644 index b4a8864faf..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml +++ /dev/null @@ -1,235 +0,0 @@ -base: - name: gb200-fp4-glm5-mtp - model: - path: glm-5-fp4 - container: "lmsysorg/sglang:v0.5.13.post1-cu130" - precision: fp4 - identity: - model: - repo: nvidia/GLM-5-NVFP4 - revision: dc54ff55a7e9e71b85db953d8bc22eca894b44c6 - frameworks: - dynamo: 1.2.1 - sglang: 0.5.13.post1 - resources: - gpu_type: gb200 - gpus_per_node: 4 - frontend: - type: dynamo - dynamo: - version: 1.2.1 - backend: - type: sglang - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_MOE_NVFP4_DISPATCH: '1' - sglang_config: - prefill: - served-model-name: GLM-5-FP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - max-running-requests: 136 - cuda-graph-max-bs: 136 - mem-fraction-static: 0.812 - context-length: 9280 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: total_tokens - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_cutlass - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: GLM-5-FP4 - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - load-balance-method: round_robin - max-running-requests: 22 - cuda-graph-max-bs: 22 - chunked-prefill-size: 64 - context-length: 9280 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - fp4-gemm-backend: flashinfer_cutlass - mem-fraction-static: 0.918 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - enable-flashinfer-allreduce-fusion: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - health_check: - max_attempts: 360 - interval_seconds: 10 - benchmark: - type: sa-bench - req_rate: inf -# ################# 8k1k ################# -zip_override_mtp_8k1k_hightpt: - name: [8k1k-1p1d_dep16, 8k1k-2p1d_dep16, 8k1k-4p1d_dep16, 8k1k-8p1d_dep16] - backend: - decode_environment: - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - sglang_config: - decode: - cuda-graph-max-bs: [91, 196, 451, 1354] - data-parallel-size: 16 - deepep-config: /configs/deepep_config.json - deepep-mode: low_latency - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - expert-parallel-size: 16 - load-balance-method: null - max-running-requests: [91, 196, 451, 1354] - mem-fraction-static: 0.858 - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - moe-runner-backend: flashinfer_cutedsl - speculative-moe-a2a-backend: deepep - speculative-moe-runner-backend: deep_gemm - tensor-parallel-size: 16 - benchmark: - concurrencies: [[109], [398], [853], [2132]] - isl: 8192 - osl: 1024 - resources: - decode_nodes: 4 - decode_workers: 1 - prefill_nodes: [1, 2, 4, 8] - prefill_workers: [1, 2, 4, 8] -zip_override_mtp_8k1k_lowlat: - name: [8k1k-1p4d_tp4, 8k1k-1p4d_tp4-conc16, 8k1k-1p8d_tp4, 8k1k-1p8d_tp4-conc4, 8k1k-1p16d_tp4, 8k1k-1p16d_tp4-conc1] - backend: - sglang_config: - decode: - cuda-graph-max-bs: [22, 15, 9, 4, 4, 1] - max-running-requests: [22, 15, 9, 4, 4, 1] - benchmark: - concurrencies: [[104], [82], [87], [47], [74], [26]] - isl: 8192 - osl: 1024 - resources: - decode_nodes: [4, 4, 8, 8, 16, 16] - decode_workers: [4, 4, 8, 8, 16, 16] - prefill_nodes: 1 - prefill_workers: 1 -# ################# 1k1k ################# -zip_override_mtp_1k1k_hightpt: - name: [1k1k-1p1d_dep16, 1k1k-1p1d_dep16-conc1024, 1k1k-1p1d_dep16-conc256, 1k1k-1p1d_dep32] - backend: - decode_environment: - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - sglang_config: - decode: - context-length: 2112 - cuda-graph-max-bs: [1627, 958, 239, 1122] - data-parallel-size: [16, 16, 16, 32] - deepep-config: /configs/deepep_config.json - deepep-mode: low_latency - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - expert-parallel-size: [16, 16, 16, 32] - max-running-requests: [1627, 958, 239, 1122] - mem-fraction-static: [0.858, 0.858, 0.858, 0.845] - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - moe-runner-backend: flashinfer_cutedsl - speculative-moe-a2a-backend: deepep - speculative-moe-runner-backend: deep_gemm - tensor-parallel-size: [16, 16, 16, 32] - prefill: - context-length: 2112 - cuda-graph-max-bs: 512 - disable-cuda-graph: true - max-running-requests: 512 - mem-fraction-static: 0.83 - benchmark: - concurrencies: [[1742], [1150], [302], [1263]] - isl: 1024 - osl: 1024 - resources: - decode_nodes: [4, 4, 4, 8] - decode_workers: 1 - prefill_nodes: 1 - prefill_workers: 1 -zip_override_mtp_1k1k_lowlat: - name: [1k1k-1p16d_tp4-conc1, 1k1k-1p16d_tp4-conc16, 1k1k-1p16d_tp4-conc4] - backend: - sglang_config: - decode: - context-length: 2112 - cuda-graph-max-bs: [1, 16, 4] - max-running-requests: [1, 16, 4] - prefill: - context-length: 2112 - cuda-graph-max-bs: 512 - disable-cuda-graph: true - max-running-requests: 512 - mem-fraction-static: 0.83 - benchmark: - concurrencies: [[26], [343], [94]] - isl: 1024 - osl: 1024 - resources: - decode_nodes: 16 - decode_workers: 16 - prefill_nodes: 1 - prefill_workers: 1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_0.yaml deleted file mode 100644 index 854b4023a5..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_0.yaml +++ /dev/null @@ -1,132 +0,0 @@ -name: gb200-fp8-glm5_1k1k_hightpt_0 - -# Combined upstream srt-slurm recipe split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention -# (see ../../../gb300-fp8/). All shared base envs and the prefill -# sglang_config are inlined here verbatim; the decode block is the shared -# base plus the topology-specific override for this concrete shape. - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.12" - precision: fp8 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 4 - prefill_workers: 2 - decode_nodes: 8 - decode_workers: 1 -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 9 -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - - sglang_config: - prefill: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - chunked-prefill-size: 131072 - max-prefill-tokens: 16384 - context-length: 2112 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - max-running-requests: 128 - cuda-graph-max-bs: 128 - mem-fraction-static: 0.7 - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - disable-radix-cache: true - - decode: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - tensor-parallel-size: 32 - data-parallel-size: 32 - expert-parallel-size: 32 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: deep_gemm - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - deepep-mode: low_latency - deepep-config: /configs/deepep_config.json - mem-fraction-static: 0.886 - context-length: 2112 - chunked-prefill-size: 64 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 1024 - osl: 1024 - concurrencies: '2161' - random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_0.yaml deleted file mode 100644 index 2ac9997129..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_0.yaml +++ /dev/null @@ -1,130 +0,0 @@ -name: gb200-fp8-glm5_1k1k_lowlat_0 - -# Combined upstream srt-slurm recipe split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention -# (see ../../../gb300-fp8/). All shared base envs and the prefill -# sglang_config are inlined here verbatim; the decode block is the shared -# base plus the topology-specific override for this concrete shape. - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.12" - precision: fp8 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - decode_nodes: 4 - decode_workers: 1 -frontend: - type: dynamo -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - - sglang_config: - prefill: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - chunked-prefill-size: 131072 - max-prefill-tokens: 16384 - context-length: 2112 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - max-running-requests: 128 - cuda-graph-max-bs: 128 - mem-fraction-static: 0.7 - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - disable-radix-cache: true - - decode: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: deep_gemm - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - deepep-mode: low_latency - deepep-config: /configs/deepep_config.json - mem-fraction-static: 0.89 - context-length: 2112 - chunked-prefill-size: 64 - max-running-requests: 1815 - cuda-graph-max-bs: 1815 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 1024 - osl: 1024 - concurrencies: '1955' - random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_1.yaml deleted file mode 100644 index 1de1730dce..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_1.yaml +++ /dev/null @@ -1,130 +0,0 @@ -name: gb200-fp8-glm5_1k1k_lowlat_1 - -# Combined upstream srt-slurm recipe split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention -# (see ../../../gb300-fp8/). All shared base envs and the prefill -# sglang_config are inlined here verbatim; the decode block is the shared -# base plus the topology-specific override for this concrete shape. - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.12" - precision: fp8 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - decode_nodes: 4 - decode_workers: 1 -frontend: - type: dynamo -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - - sglang_config: - prefill: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - chunked-prefill-size: 131072 - max-prefill-tokens: 16384 - context-length: 2112 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - max-running-requests: 128 - cuda-graph-max-bs: 128 - mem-fraction-static: 0.7 - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - disable-radix-cache: true - - decode: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: deep_gemm - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - deepep-mode: low_latency - deepep-config: /configs/deepep_config.json - mem-fraction-static: 0.89 - context-length: 2112 - chunked-prefill-size: 64 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 1024 - osl: 1024 - concurrencies: '1170' - random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_2.yaml deleted file mode 100644 index 4efada1878..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_2.yaml +++ /dev/null @@ -1,130 +0,0 @@ -name: gb200-fp8-glm5_1k1k_lowlat_2 - -# Combined upstream srt-slurm recipe split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention -# (see ../../../gb300-fp8/). All shared base envs and the prefill -# sglang_config are inlined here verbatim; the decode block is the shared -# base plus the topology-specific override for this concrete shape. - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.12" - precision: fp8 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - decode_nodes: 4 - decode_workers: 1 -frontend: - type: dynamo -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - - sglang_config: - prefill: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - chunked-prefill-size: 131072 - max-prefill-tokens: 16384 - context-length: 2112 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - max-running-requests: 128 - cuda-graph-max-bs: 128 - mem-fraction-static: 0.7 - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - disable-radix-cache: true - - decode: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: deep_gemm - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - deepep-mode: low_latency - deepep-config: /configs/deepep_config.json - mem-fraction-static: 0.89 - context-length: 2112 - chunked-prefill-size: 64 - max-running-requests: 256 - cuda-graph-max-bs: 256 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 1024 - osl: 1024 - concurrencies: '298' - random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_3.yaml deleted file mode 100644 index 23d65f5c5e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_3.yaml +++ /dev/null @@ -1,121 +0,0 @@ -name: gb200-fp8-glm5_1k1k_lowlat_3 - -# Combined upstream srt-slurm recipe split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention -# (see ../../../gb300-fp8/). All shared base envs and the prefill -# sglang_config are inlined here verbatim; the decode block is the shared -# base plus the topology-specific override for this concrete shape. - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.12" - precision: fp8 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - decode_nodes: 12 - decode_workers: 6 -frontend: - type: dynamo -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - sglang_config: - prefill: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - chunked-prefill-size: 131072 - max-prefill-tokens: 16384 - context-length: 2112 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - max-running-requests: 128 - cuda-graph-max-bs: 128 - mem-fraction-static: 0.7 - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - disable-radix-cache: true - - decode: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-flashinfer-allreduce-fusion: false - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - mem-fraction-static: 0.942 - context-length: 2112 - chunked-prefill-size: 64 - max-running-requests: 1 - cuda-graph-max-bs: 1 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 1024 - osl: 1024 - concurrencies: '8' - random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_4.yaml deleted file mode 100644 index 784ced1656..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_4.yaml +++ /dev/null @@ -1,121 +0,0 @@ -name: gb200-fp8-glm5_1k1k_lowlat_4 - -# Combined upstream srt-slurm recipe split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention -# (see ../../../gb300-fp8/). All shared base envs and the prefill -# sglang_config are inlined here verbatim; the decode block is the shared -# base plus the topology-specific override for this concrete shape. - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.12" - precision: fp8 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - decode_nodes: 12 - decode_workers: 6 -frontend: - type: dynamo -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - sglang_config: - prefill: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - chunked-prefill-size: 131072 - max-prefill-tokens: 16384 - context-length: 2112 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - max-running-requests: 128 - cuda-graph-max-bs: 128 - mem-fraction-static: 0.7 - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - disable-radix-cache: true - - decode: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-flashinfer-allreduce-fusion: false - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - mem-fraction-static: 0.942 - context-length: 2112 - chunked-prefill-size: 64 - max-running-requests: 16 - cuda-graph-max-bs: 16 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 1024 - osl: 1024 - concurrencies: '72' - random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_5.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_5.yaml deleted file mode 100644 index f611c7ad62..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_5.yaml +++ /dev/null @@ -1,121 +0,0 @@ -name: gb200-fp8-glm5_1k1k_lowlat_5 - -# Combined upstream srt-slurm recipe split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention -# (see ../../../gb300-fp8/). All shared base envs and the prefill -# sglang_config are inlined here verbatim; the decode block is the shared -# base plus the topology-specific override for this concrete shape. - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.12" - precision: fp8 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - decode_nodes: 12 - decode_workers: 6 -frontend: - type: dynamo -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - sglang_config: - prefill: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - chunked-prefill-size: 131072 - max-prefill-tokens: 16384 - context-length: 2112 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - max-running-requests: 128 - cuda-graph-max-bs: 128 - mem-fraction-static: 0.7 - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - disable-radix-cache: true - - decode: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-flashinfer-allreduce-fusion: false - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - mem-fraction-static: 0.942 - context-length: 2112 - chunked-prefill-size: 64 - max-running-requests: 4 - cuda-graph-max-bs: 4 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 1024 - osl: 1024 - concurrencies: '20' - random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_0.yaml deleted file mode 100644 index 80b88d7964..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_0.yaml +++ /dev/null @@ -1,132 +0,0 @@ -name: gb200-fp8-glm5_8k1k_hightpt_0 - -# Combined upstream srt-slurm recipe split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention -# (see ../../../gb300-fp8/). All shared base envs and the prefill -# sglang_config are inlined here verbatim; the decode block is the shared -# base plus the topology-specific override for this concrete shape. - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.12" - precision: fp8 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 6 - prefill_workers: 3 - decode_nodes: 8 - decode_workers: 1 -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 9 -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - - sglang_config: - prefill: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - chunked-prefill-size: 131072 - max-prefill-tokens: 16384 - context-length: 9280 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - max-running-requests: 128 - cuda-graph-max-bs: 128 - mem-fraction-static: 0.7 - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - disable-radix-cache: true - - decode: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - tensor-parallel-size: 32 - data-parallel-size: 32 - expert-parallel-size: 32 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: deep_gemm - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - deepep-mode: low_latency - deepep-config: /configs/deepep_config.json - mem-fraction-static: 0.89 - context-length: 9280 - chunked-prefill-size: 64 - max-running-requests: 256 - cuda-graph-max-bs: 256 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 8192 - osl: 1024 - concurrencies: '519' - random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_1.yaml deleted file mode 100644 index 734aec020f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_1.yaml +++ /dev/null @@ -1,132 +0,0 @@ -name: gb200-fp8-glm5_8k1k_hightpt_1 - -# Combined upstream srt-slurm recipe split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention -# (see ../../../gb300-fp8/). All shared base envs and the prefill -# sglang_config are inlined here verbatim; the decode block is the shared -# base plus the topology-specific override for this concrete shape. - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.12" - precision: fp8 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 8 - prefill_workers: 4 - decode_nodes: 4 - decode_workers: 1 -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 9 -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - - sglang_config: - prefill: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - chunked-prefill-size: 131072 - max-prefill-tokens: 16384 - context-length: 9280 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - max-running-requests: 128 - cuda-graph-max-bs: 128 - mem-fraction-static: 0.7 - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - disable-radix-cache: true - - decode: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: deep_gemm - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - deepep-mode: low_latency - deepep-config: /configs/deepep_config.json - mem-fraction-static: 0.89 - context-length: 9280 - chunked-prefill-size: 64 - max-running-requests: 802 - cuda-graph-max-bs: 802 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 8192 - osl: 1024 - concurrencies: '1484' - random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_2.yaml deleted file mode 100644 index feb1c19b73..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_2.yaml +++ /dev/null @@ -1,132 +0,0 @@ -name: gb200-fp8-glm5_8k1k_hightpt_2 - -# Combined upstream srt-slurm recipe split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention -# (see ../../../gb300-fp8/). All shared base envs and the prefill -# sglang_config are inlined here verbatim; the decode block is the shared -# base plus the topology-specific override for this concrete shape. - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.12" - precision: fp8 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 10 - prefill_workers: 5 - decode_nodes: 8 - decode_workers: 1 -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 9 -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - - sglang_config: - prefill: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - chunked-prefill-size: 131072 - max-prefill-tokens: 16384 - context-length: 9280 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - max-running-requests: 128 - cuda-graph-max-bs: 128 - mem-fraction-static: 0.7 - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - disable-radix-cache: true - - decode: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - tensor-parallel-size: 32 - data-parallel-size: 32 - expert-parallel-size: 32 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: deep_gemm - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - deepep-mode: low_latency - deepep-config: /configs/deepep_config.json - mem-fraction-static: 0.89 - context-length: 9280 - chunked-prefill-size: 64 - max-running-requests: 828 - cuda-graph-max-bs: 828 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 8192 - osl: 1024 - concurrencies: '1688' - random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_3.yaml deleted file mode 100644 index 29da98c953..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_3.yaml +++ /dev/null @@ -1,132 +0,0 @@ -name: gb200-fp8-glm5_8k1k_hightpt_3 - -# Combined upstream srt-slurm recipe split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention -# (see ../../../gb300-fp8/). All shared base envs and the prefill -# sglang_config are inlined here verbatim; the decode block is the shared -# base plus the topology-specific override for this concrete shape. - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.12" - precision: fp8 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 14 - prefill_workers: 7 - decode_nodes: 4 - decode_workers: 1 -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 9 -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - - sglang_config: - prefill: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - chunked-prefill-size: 131072 - max-prefill-tokens: 16384 - context-length: 9280 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - max-running-requests: 128 - cuda-graph-max-bs: 128 - mem-fraction-static: 0.7 - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - disable-radix-cache: true - - decode: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: deep_gemm - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - deepep-mode: low_latency - deepep-config: /configs/deepep_config.json - mem-fraction-static: 0.89 - context-length: 9280 - chunked-prefill-size: 64 - max-running-requests: 1514 - cuda-graph-max-bs: 1514 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 8192 - osl: 1024 - concurrencies: '2699' - random_range_ratio: 1.0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml deleted file mode 100644 index 385063085c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml +++ /dev/null @@ -1,119 +0,0 @@ -name: gb200-fp8-glm5_8k1k_lowlat_0 - -# Combined upstream srt-slurm recipe split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention -# (see ../../../gb300-fp8/). All shared base envs and the prefill -# sglang_config are inlined here verbatim; the decode block is the shared -# base plus the topology-specific override for this concrete shape. - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.12" - precision: fp8 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - decode_nodes: 16 - decode_workers: 8 -frontend: - type: dynamo -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - sglang_config: - prefill: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - chunked-prefill-size: 131072 - max-prefill-tokens: 16384 - context-length: 9280 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - max-running-requests: 128 - cuda-graph-max-bs: 128 - mem-fraction-static: 0.7 - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-flashinfer-allreduce-fusion: false - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - mem-fraction-static: 0.943 - context-length: 9280 - chunked-prefill-size: 64 - max-running-requests: 1 - cuda-graph-max-bs: 1 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 8192 - osl: 1024 - concurrencies: '8' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml deleted file mode 100644 index 035dda552d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml +++ /dev/null @@ -1,119 +0,0 @@ -name: gb200-fp8-glm5_8k1k_lowlat_1 - -# Combined upstream srt-slurm recipe split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention -# (see ../../../gb300-fp8/). All shared base envs and the prefill -# sglang_config are inlined here verbatim; the decode block is the shared -# base plus the topology-specific override for this concrete shape. - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.12" - precision: fp8 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - decode_nodes: 16 - decode_workers: 8 -frontend: - type: dynamo -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - sglang_config: - prefill: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - chunked-prefill-size: 131072 - max-prefill-tokens: 16384 - context-length: 9280 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - max-running-requests: 128 - cuda-graph-max-bs: 128 - mem-fraction-static: 0.7 - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-flashinfer-allreduce-fusion: false - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - mem-fraction-static: 0.943 - context-length: 9280 - chunked-prefill-size: 64 - max-running-requests: 16 - cuda-graph-max-bs: 16 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 8192 - osl: 1024 - concurrencies: '90' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml deleted file mode 100644 index d2dadc3fa3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml +++ /dev/null @@ -1,119 +0,0 @@ -name: gb200-fp8-glm5_8k1k_lowlat_2 - -# Combined upstream srt-slurm recipe split into one flat yaml per concrete -# topology to match the InferenceX glm5 sglang convention -# (see ../../../gb300-fp8/). All shared base envs and the prefill -# sglang_config are inlined here verbatim; the decode block is the shared -# base plus the topology-specific override for this concrete shape. - -model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.12" - precision: fp8 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - decode_nodes: 16 - decode_workers: 8 -frontend: - type: dynamo -dynamo: - version: 1.2.1 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - - sglang_config: - prefill: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - enable-flashinfer-allreduce-fusion: false - chunked-prefill-size: 131072 - max-prefill-tokens: 16384 - context-length: 9280 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - max-running-requests: 128 - cuda-graph-max-bs: 128 - mem-fraction-static: 0.7 - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - served-model-name: GLM-5.1-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-flashinfer-allreduce-fusion: false - nsa-decode-backend: trtllm - nsa-prefill-backend: trtllm - moe-runner-backend: flashinfer_trtllm - mem-fraction-static: 0.943 - context-length: 9280 - chunked-prefill-size: 64 - max-running-requests: 4 - cuda-graph-max-bs: 4 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - req_rate: inf - isl: 8192 - osl: 1024 - concurrencies: '9' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml deleted file mode 100644 index 372c794079..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml +++ /dev/null @@ -1,159 +0,0 @@ -base: - name: gb200-fp8-glm5-mtp - model: - path: glm-5.1-fp8 - container: "lmsysorg/sglang:v0.5.13.post1-cu130" - precision: fp8 - resources: - gpu_type: gb200 - gpus_per_node: 4 - frontend: - type: dynamo - dynamo: - version: 1.2.1 - backend: - type: sglang - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - PYTHONUNBUFFERED: '1' - DYN_SKIP_SGLANG_LOG_FORMATTING: '1' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - MC_TE_METRIC: 'true' - MC_FORCE_MNNVL: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - DYN_REQUEST_PLANE: nats - sglang_config: - prefill: - served-model-name: GLM-5-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: prefill - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - chunked-prefill-size: 131072 - max-prefill-tokens: 16384 - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - enable-flashinfer-allreduce-fusion: false - max-running-requests: 136 - cuda-graph-max-bs: 136 - mem-fraction-static: 0.7 - context-length: 9280 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - decode: - served-model-name: GLM-5-FP8 - trust-remote-code: true - quantization: fp8 - kv-cache-dtype: fp8_e4m3 - disaggregation-mode: decode - disaggregation-transfer-backend: nixl - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - nsa-prefill-backend: trtllm - nsa-decode-backend: trtllm - moe-runner-backend: flashinfer_trtllm - enable-flashinfer-allreduce-fusion: false - speculative-algorithm: EAGLE - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - max-running-requests: 28 - cuda-graph-max-bs: 28 - chunked-prefill-size: 64 - mem-fraction-static: 0.94 - context-length: 9280 - skip-tokenizer-init: true - stream-interval: 30 - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - health_check: - max_attempts: 360 - interval_seconds: 10 - benchmark: - type: sa-bench - req_rate: inf - # MTP: EAGLE-style spec decoding is trained against chat-formatted - # inputs — keep the chat template on explicitly rather than relying - # on the srtctl schema default. - use_chat_template: true -# ################# 8k1k ################# -zip_override_mtp_8k1k_hightpt: - name: [8k1k-1p1d_dep16, 8k1k-2p1d_dep16, 8k1k-4p1d_dep16] - backend: - decode_environment: - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - sglang_config: - decode: - cuda-graph-max-bs: [111, 234, 535] - data-parallel-size: 16 - deepep-config: /configs/deepep_config.json - deepep-mode: low_latency - enable-dp-attention: true - enable-dp-lm-head: true - ep-dispatch-algorithm: static - ep-num-redundant-experts: 0 - expert-parallel-size: 16 - max-running-requests: [111, 234, 535] - mem-fraction-static: 0.85 - moe-a2a-backend: deepep - moe-dense-tp-size: 1 - moe-runner-backend: deep_gemm - tensor-parallel-size: 16 - benchmark: - concurrencies: [[137], [620], [1305]] - isl: 8192 - osl: 1024 - resources: - decode_nodes: 4 - decode_workers: 1 - prefill_nodes: [2, 4, 8] - prefill_workers: [1, 2, 4] -zip_override_mtp_8k1k_lowlat: - name: [8k1k-1p4d_tp8, 8k1k-1p4d_tp8-conc56, 8k1k-1p8d_tp8, 8k1k-1p8d_tp8-conc12, 8k1k-1p8d_tp8-conc31] - backend: - sglang_config: - decode: - cuda-graph-max-bs: [28, 8, 12, 1, 2] - max-running-requests: [28, 8, 12, 1, 2] - benchmark: - concurrencies: [[136], [56], [120], [12], [31]] - isl: 8192 - osl: 1024 - resources: - decode_nodes: [8, 8, 16, 16, 16] - decode_workers: [4, 4, 8, 8, 8] - prefill_nodes: 2 - prefill_workers: 1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml deleted file mode 100644 index 31ac6edf8e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml +++ /dev/null @@ -1,173 +0,0 @@ -name: "gb300-fp4-glm5_8k1k_lowlat_0" - -# Ported from upstream srt-slurm recipes/gb300-fp4/glm5.yaml (PR #152). -# Upstream uses a single combined file with `zip_override_*` arrays -# expanded by srtctl across zip indices. We split into one flat yaml -# per concrete topology to match the InferenceX dsv4 sglang convention -# (see ../deepseek-v4/8k1k/*.yaml). All shared base envs and the -# prefill sglang_config are inlined here verbatim from the upstream -# `base:` block; the decode block is the upstream base plus the -# topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -# Released dynamo wheel; unlike hash-based sources, this recipe does not -# require a persistent /configs/dynamo-wheels build cache. -dynamo: - version: "1.1.0" - -slurm: - time_limit: "03:00:00" - -# Mirror dsv4 sglang recipes: cpus-per-task=144 avoids the 1-CPU -# default that turns dynamo install + sglang weight load into a serial -# crawl; mem=0 grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 3 - decode_workers: 3 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - - # Other flags - # disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_cutedsl" - fp4-gemm-backend: "flashinfer_cutlass" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - # disable-shared-experts-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - # Parallelism (override from upstream zip_override_*_lowlat) - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - enable-flashinfer-allreduce-fusion: true - - moe-runner-backend: "flashinfer_trtllm" - max-running-requests: 128 - cuda-graph-max-bs: 128 - - - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml deleted file mode 100644 index 95ffd216b7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml +++ /dev/null @@ -1,173 +0,0 @@ -name: "gb300-fp4-glm5_8k1k_lowlat_1" - -# Ported from upstream srt-slurm recipes/gb300-fp4/glm5.yaml (PR #152). -# Upstream uses a single combined file with `zip_override_*` arrays -# expanded by srtctl across zip indices. We split into one flat yaml -# per concrete topology to match the InferenceX dsv4 sglang convention -# (see ../deepseek-v4/8k1k/*.yaml). All shared base envs and the -# prefill sglang_config are inlined here verbatim from the upstream -# `base:` block; the decode block is the upstream base plus the -# topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -# Released dynamo wheel; unlike hash-based sources, this recipe does not -# require a persistent /configs/dynamo-wheels build cache. -dynamo: - version: "1.1.0" - -slurm: - time_limit: "03:00:00" - -# Mirror dsv4 sglang recipes: cpus-per-task=144 avoids the 1-CPU -# default that turns dynamo install + sglang weight load into a serial -# crawl; mem=0 grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 5 - decode_workers: 5 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - - # Other flags - # disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_cutedsl" - fp4-gemm-backend: "flashinfer_cutlass" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - # disable-shared-experts-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - # Parallelism (override from upstream zip_override_*_lowlat) - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - enable-flashinfer-allreduce-fusion: true - - moe-runner-backend: "flashinfer_trtllm" - max-running-requests: 64 - cuda-graph-max-bs: 64 - - - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "64" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml deleted file mode 100644 index 506dede54f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml +++ /dev/null @@ -1,173 +0,0 @@ -name: "gb300-fp4-glm5_8k1k_lowlat_2" - -# Ported from upstream srt-slurm recipes/gb300-fp4/glm5.yaml (PR #152). -# Upstream uses a single combined file with `zip_override_*` arrays -# expanded by srtctl across zip indices. We split into one flat yaml -# per concrete topology to match the InferenceX dsv4 sglang convention -# (see ../deepseek-v4/8k1k/*.yaml). All shared base envs and the -# prefill sglang_config are inlined here verbatim from the upstream -# `base:` block; the decode block is the upstream base plus the -# topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -# Released dynamo wheel; unlike hash-based sources, this recipe does not -# require a persistent /configs/dynamo-wheels build cache. -dynamo: - version: "1.1.0" - -slurm: - time_limit: "03:00:00" - -# Mirror dsv4 sglang recipes: cpus-per-task=144 avoids the 1-CPU -# default that turns dynamo install + sglang weight load into a serial -# crawl; mem=0 grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 9 - decode_workers: 9 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - - # Other flags - # disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_cutedsl" - fp4-gemm-backend: "flashinfer_cutlass" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - # disable-shared-experts-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - # Parallelism (override from upstream zip_override_*_lowlat) - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - enable-flashinfer-allreduce-fusion: true - - moe-runner-backend: "flashinfer_trtllm" - max-running-requests: 32 - cuda-graph-max-bs: 32 - - - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "32" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml deleted file mode 100644 index 43e96c0a32..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml +++ /dev/null @@ -1,173 +0,0 @@ -name: "gb300-fp4-glm5_8k1k_lowlat_3" - -# Ported from upstream srt-slurm recipes/gb300-fp4/glm5.yaml (PR #152). -# Upstream uses a single combined file with `zip_override_*` arrays -# expanded by srtctl across zip indices. We split into one flat yaml -# per concrete topology to match the InferenceX dsv4 sglang convention -# (see ../deepseek-v4/8k1k/*.yaml). All shared base envs and the -# prefill sglang_config are inlined here verbatim from the upstream -# `base:` block; the decode block is the upstream base plus the -# topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -# Released dynamo wheel; unlike hash-based sources, this recipe does not -# require a persistent /configs/dynamo-wheels build cache. -dynamo: - version: "1.1.0" - -slurm: - time_limit: "03:00:00" - -# Mirror dsv4 sglang recipes: cpus-per-task=144 avoids the 1-CPU -# default that turns dynamo install + sglang weight load into a serial -# crawl; mem=0 grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 15 - decode_workers: 15 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - - # Other flags - # disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_cutedsl" - fp4-gemm-backend: "flashinfer_cutlass" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - # disable-shared-experts-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - # Parallelism (override from upstream zip_override_*_lowlat) - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - enable-flashinfer-allreduce-fusion: true - - moe-runner-backend: "flashinfer_trtllm" - max-running-requests: 16 - cuda-graph-max-bs: 16 - - - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml deleted file mode 100644 index 2bcc483e9f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml +++ /dev/null @@ -1,173 +0,0 @@ -name: "gb300-fp4-glm5_8k1k_lowlat_4" - -# Ported from upstream srt-slurm recipes/gb300-fp4/glm5.yaml (PR #152). -# Upstream uses a single combined file with `zip_override_*` arrays -# expanded by srtctl across zip indices. We split into one flat yaml -# per concrete topology to match the InferenceX dsv4 sglang convention -# (see ../deepseek-v4/8k1k/*.yaml). All shared base envs and the -# prefill sglang_config are inlined here verbatim from the upstream -# `base:` block; the decode block is the upstream base plus the -# topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -# Released dynamo wheel; unlike hash-based sources, this recipe does not -# require a persistent /configs/dynamo-wheels build cache. -dynamo: - version: "1.1.0" - -slurm: - time_limit: "03:00:00" - -# Mirror dsv4 sglang recipes: cpus-per-task=144 avoids the 1-CPU -# default that turns dynamo install + sglang weight load into a serial -# crawl; mem=0 grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 17 - decode_workers: 17 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - - # Other flags - # disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_cutedsl" - fp4-gemm-backend: "flashinfer_cutlass" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - # disable-shared-experts-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - # Parallelism (override from upstream zip_override_*_lowlat) - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - enable-flashinfer-allreduce-fusion: true - - moe-runner-backend: "flashinfer_trtllm" - max-running-requests: 1 - cuda-graph-max-bs: 1 - - - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "12" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache-cap48.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache-cap48.yaml deleted file mode 100644 index 66371afc2d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache-cap48.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-cap48 - -model: { path: qwen3.5-fp4, container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b, precision: fp4 } -identity: - model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 } - container: { image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b } -slurm: { time_limit: "8:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: { gpu_type: gb200, gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 2 } -infra: { nats_max_payload_mb: 8 } -dynamo: - install: true - request_plane: tcp - wheel: 1.4.2 -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -backend: - type: sglang - aggregated_environment: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false - enable-symm-mem: false - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-radix-cache-strategy: extra_buffer_lazy - mamba-track-interval: 1048576 - attention-backend: trtllm_mha - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 8 - cuda-graph-max-bs: 48 - max-running-requests: 48 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.88 - max-mamba-cache-size: 192 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 2.5 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - hicache-write-policy: write_through_selective - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - watchdog-timeout: 1000000 - enable-metrics: true - enable-cache-report: true - -sbatch_directives: { mem: "0", cpus-per-task: "144" } -srun_options: { mem: "0", container-remap-root: "" } -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "2" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache-k3-baseline.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache-k3-baseline.yaml deleted file mode 100644 index 74db89add6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache-k3-baseline.yaml +++ /dev/null @@ -1,108 +0,0 @@ -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-k3-baseline - -model: { path: qwen3.5-fp4, container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b, precision: fp4 } -identity: - model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 } - container: { image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b } -slurm: { time_limit: "8:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: { gpu_type: gb200, gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 2 } -infra: { nats_max_payload_mb: 8 } -dynamo: - install: true - request_plane: tcp - wheel: 1.4.2 -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -backend: - type: sglang - aggregated_environment: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false - enable-symm-mem: false - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-radix-cache-strategy: extra_buffer_lazy - mamba-track-interval: 1048576 - attention-backend: trtllm_mha - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 8 - cuda-graph-max-bs: 64 - max-running-requests: 80 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.88 - max-mamba-cache-size: 320 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 1 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 2.5 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - hicache-write-policy: write_through_selective - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - watchdog-timeout: 1000000 - enable-metrics: true - enable-cache-report: true - -sbatch_directives: { mem: "0", cpus-per-task: "144" } -srun_options: { mem: "0", container-remap-root: "" } -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "2" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache.yaml deleted file mode 100644 index d2603a5de4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache.yaml +++ /dev/null @@ -1,109 +0,0 @@ -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp2ep2-hicache-k5 - -model: { path: qwen3.5-fp4, container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b, precision: fp4 } -identity: - model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 } - container: { image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b } -slurm: { time_limit: "8:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: { gpu_type: gb200, gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 2 } -infra: { nats_max_payload_mb: 8 } -dynamo: - install: true - request_plane: tcp - wheel: 1.4.2 -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -backend: - type: sglang - aggregated_environment: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false - enable-symm-mem: false - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-radix-cache-strategy: extra_buffer_lazy - mamba-track-interval: 1048576 - attention-backend: trtllm_mha - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 8 - cuda-graph-max-bs: 64 - max-running-requests: 80 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.88 - # extra_buffer_lazy uses four physical Mamba state slots per running request. - # This layout is the measured K5 middle frontier through C28. - max-mamba-cache-size: 320 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 2.5 - hicache-io-backend: direct - hicache-mem-layout: page_first_direct - hicache-write-policy: write_through_selective - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - watchdog-timeout: 1000000 - enable-metrics: true - enable-cache-report: true - -sbatch_directives: { mem: "0", cpus-per-task: "144" } -srun_options: { mem: "0", container-remap-root: "" } -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "2" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-hicache.yaml deleted file mode 100644 index 917f7e5765..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-hicache.yaml +++ /dev/null @@ -1,85 +0,0 @@ -name: qwen35-gb200-sglang-agentic-mtp-agg-tp4-hicache - -model: { path: qwen3.5-fp4, container: dynamo-sglang, precision: fp4 } -identity: - model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 } - container: { image: lmsysorg/sglang:v0.5.17-cu130 } -slurm: { time_limit: "8:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: { gpu_type: gb200, gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } -infra: { nats_max_payload_mb: 8 } -frontend: - type: sglang - args: - worker-startup-timeout-secs: 3600 - -backend: - type: sglang - aggregated_environment: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 64 - max-running-requests: 160 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.78 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.70 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 - enable-metrics: true - enable-cache-report: true - -sbatch_directives: { mem: "0", cpus-per-task: "144" } -srun_options: { mem: "0", container-remap-root: "" } -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "4" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-no-symm.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-no-symm.yaml deleted file mode 100644 index 8a44755342..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-no-symm.yaml +++ /dev/null @@ -1,106 +0,0 @@ -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5-no-symm - -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b - precision: fp4 - -identity: - model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 } - container: { image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b } - -slurm: { time_limit: "8:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } - -resources: - gpu_type: gb200 - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: { nats_max_payload_mb: 8 } -dynamo: - install: true - request_plane: tcp - wheel: 1.4.2 -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -backend: - type: sglang - aggregated_environment: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: false - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - cuda-graph-max-bs: 64 - max-running-requests: 80 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.80 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - enable-metrics: true - enable-cache-report: true - -sbatch_directives: { mem: "0", cpus-per-task: "144" } -srun_options: { mem: "0", container-remap-root: "" } - -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "4" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-parity.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-parity.yaml deleted file mode 100644 index 704e331e56..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-parity.yaml +++ /dev/null @@ -1,109 +0,0 @@ -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5-parity - -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b - precision: fp4 - -identity: - model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 } - container: { image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b } - -slurm: { time_limit: "8:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } - -resources: - gpu_type: gb200 - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: { nats_max_payload_mb: 8 } -dynamo: - install: true - request_plane: tcp - wheel: 1.4.2 -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -backend: - type: sglang - aggregated_environment: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer_lazy - mamba-track-interval: 1048576 - attention-backend: trtllm_mha - linear-attn-prefill-backend: flashinfer - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 8 - cuda-graph-max-bs: 64 - max-running-requests: 80 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.80 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - enable-metrics: true - enable-cache-report: true - -sbatch_directives: { mem: "0", cpus-per-task: "144" } -srun_options: { mem: "0", container-remap-root: "" } - -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "4" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp.yaml deleted file mode 100644 index 8ef6fca24f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp.yaml +++ /dev/null @@ -1,106 +0,0 @@ -name: qwen35-gb200-dynsg-agentic-mtp-agg-tp4-k5 - -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-20260818-c0b6474b - precision: fp4 - -identity: - model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 } - container: { image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b } - -slurm: { time_limit: "8:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } - -resources: - gpu_type: gb200 - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: { nats_max_payload_mb: 8 } -dynamo: - install: true - request_plane: tcp - wheel: 1.4.2 -environment: - PIP_BREAK_SYSTEM_PACKAGES: "1" -frontend: - type: dynamo - enable_multiple_frontends: false - env: {} - args: - router-mode: round-robin - router-session-affinity-ttl-secs: 3600 - -backend: - type: sglang - aggregated_environment: - PYTHONNOUSERSITE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_ENABLE_SPEC_V2: "1" - SGL_ENABLE_JIT_DEEPGEMM: "false" - TORCH_CUDA_ARCH_LIST: "10.0" - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - fp4-gemm-backend: flashinfer_cutlass - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - linear-attn-decode-backend: flashinfer - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - cuda-graph-max-bs: 64 - max-running-requests: 80 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.80 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - mamba-max-states-per-path: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - weight-loader-drop-cache-after-load: false - model-loader-extra-config: '{"enable_multithread_load":true}' - enable-metrics: true - enable-cache-report: true - -sbatch_directives: { mem: "0", cpus-per-task: "144" } -srun_options: { mem: "0", container-remap-root: "" } - -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "false" - TP: "4" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" - AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"enable_thinking":true},"presence_penalty":0,"temperature":0.6,"top_k":20,"top_p":0.95}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-dep4-dep16.yaml deleted file mode 100644 index 49ead5b703..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-dep4-dep16.yaml +++ /dev/null @@ -1,168 +0,0 @@ -name: "qwen3.5-1p1d-dep4-dep16" - -setup_script: rebuild-deepep.sh - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 4096 - mem-fraction-static: 0.80 - max-mamba-cache-size: 2048 - max-running-requests: 2048 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "512x1024x2048" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-tp4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-tp4-tp4.yaml deleted file mode 100644 index 7decea8649..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-tp4-tp4.yaml +++ /dev/null @@ -1,136 +0,0 @@ -name: "qwen3.5-1p1d-tp4-tp4" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_HEALTH_CHECK_TIMEOUT: "3600" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 4096 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "prefill" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 4096 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "decode" - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 10 - num_warmup_mult: 1 - random_range_ratio: 0.8 - concurrencies: "1x2x4x8x16x32x64" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/2p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/2p1d-dep4-dep16.yaml deleted file mode 100644 index c5764de735..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/2p1d-dep4-dep16.yaml +++ /dev/null @@ -1,168 +0,0 @@ -name: "qwen3.5-2p1d-dep4-dep16" - -setup_script: rebuild-deepep.sh - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 2 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 4 - prefill_workers: 2 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 8192 - mem-fraction-static: 0.75 - max-mamba-cache-size: 4096 - max-running-requests: 4096 - cuda-graph-max-bs: 256 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "4096" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml deleted file mode 100644 index cc9a7335f5..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml +++ /dev/null @@ -1,134 +0,0 @@ -name: "qwen3.5-1p1d-tp4-tp4" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_HEALTH_CHECK_TIMEOUT: "3600" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 16384 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "prefill" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 16384 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "decode" - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "1x2x4x8x16x32x64x128" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/4p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/4p1d-dep4-dep16.yaml deleted file mode 100644 index f52bf9df51..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/4p1d-dep4-dep16.yaml +++ /dev/null @@ -1,173 +0,0 @@ -name: "qwen3.5-4p1d-dep4-dep16" - -setup_script: rebuild-deepep.sh - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 4 - prefill_workers: 4 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 16384 - mem-fraction-static: 0.80 - max-mamba-cache-size: 2048 - max-running-requests: 2048 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 20 - num_warmup_mult: 2 - random_range_ratio: 0.8 - concurrencies: "1024" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/8p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/8p1d-dep4-dep16.yaml deleted file mode 100644 index 58f2604a78..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/8p1d-dep4-dep16.yaml +++ /dev/null @@ -1,161 +0,0 @@ -name: "qwen3.5-8p1d-dep4-dep16" - -setup_script: rebuild-deepep.sh - -sbatch_directives: - mem: "0" - -infra: - etcd_nats_dedicated_node: true - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 8 - decode_nodes: 4 - prefill_workers: 8 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 16384 - mem-fraction-static: 0.80 - max-mamba-cache-size: 2048 - max-running-requests: 2048 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "2048x4096" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml deleted file mode 100644 index 5e7b9425ed..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml +++ /dev/null @@ -1,156 +0,0 @@ -# Qwen3.5 FP8 GB200 disaggregated MTP 1P1D TP4/TP4 topology. - -name: "qwen3.5-fp8-gb200-mtp-8k1k-1p1d-tp4-tp4" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 16384 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "prefill" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - max-mamba-cache-size: 256 - moe-runner-backend: "flashinfer_trtllm" - - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - disable-radix-cache: true - max-running-requests: 128 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 16384 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "decode" - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "1x2x8" - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml deleted file mode 100644 index 3de02a24c7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml +++ /dev/null @@ -1,170 +0,0 @@ -# Qwen3.5 FP8 GB200 disaggregated MTP 1P1D TEP8/TEP8 points. - -name: "qwen3.5-fp8-gb200-mtp-8k1k-1p1d-tep8-tep8" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 8 - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - max-total-tokens: 128000 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 9236 - cuda-graph-max-bs: 320 - scheduler-recv-interval: 10 - decode-log-interval: 50 - stream-interval: 50 - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "mooncake" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 8 - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - max-mamba-cache-size: 1024 - moe-runner-backend: "flashinfer_trtllm" - - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - max-total-tokens: 2200000 - chunked-prefill-size: 4096 - max-prefill-tokens: 16384 - context-length: 9236 - cuda-graph-max-bs: 320 - scheduler-recv-interval: 10 - decode-log-interval: 50 - stream-interval: 50 - disaggregation-mode: "decode" - disaggregation-transfer-backend: "mooncake" - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 20 - num_warmup_mult: 2 - random_range_ratio: 0.8 - concurrencies: "32x48x80" - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_0.yaml deleted file mode 100644 index 7895cc7fc4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_0.yaml +++ /dev/null @@ -1,186 +0,0 @@ -# Qwen3.5 FP8 GB200 disaggregated MTP 3P1D DEP4/DEP16 point. - -name: "qwen3.5-fp8-gb200-mtp-8k1k-3p1d-dep4-dep16" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 2 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 4 - prefill_workers: 3 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.7 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 16384 - mem-fraction-static: 0.7 - max-mamba-cache-size: 1024 - max-running-requests: 1024 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 20 - num_warmup_mult: 2 - random_range_ratio: 0.8 - concurrencies: "480" - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_1.yaml deleted file mode 100644 index 1f888c5220..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_1.yaml +++ /dev/null @@ -1,186 +0,0 @@ -# Qwen3.5 FP8 GB200 disaggregated MTP 4P1D DEP4/DEP16 point. - -name: "qwen3.5-fp8-gb200-mtp-8k1k-4p1d-dep4-dep16" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 4 - prefill_workers: 4 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.7 - chunked-prefill-size: 98304 - max-prefill-tokens: 24576 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 16384 - mem-fraction-static: 0.7 - max-mamba-cache-size: 1024 - max-running-requests: 1024 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 20 - num_warmup_mult: 2 - random_range_ratio: 0.8 - concurrencies: "768" - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_2.yaml deleted file mode 100644 index 55220a08f4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_2.yaml +++ /dev/null @@ -1,186 +0,0 @@ -# Qwen3.5 FP8 GB200 disaggregated MTP 6P1D DEP4/DEP16 point. - -name: "qwen3.5-fp8-gb200-mtp-8k1k-6p1d-dep4-dep16" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 5 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 6 - decode_nodes: 4 - prefill_workers: 6 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31001 - mem-fraction-static: 0.7 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31001 - - chunked-prefill-size: 4096 - context-length: 16384 - mem-fraction-static: 0.7 - max-mamba-cache-size: 1024 - max-running-requests: 1024 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 20 - num_warmup_mult: 2 - random_range_ratio: 0.8 - concurrencies: "1280" - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_3.yaml deleted file mode 100644 index ba2272058b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_3.yaml +++ /dev/null @@ -1,186 +0,0 @@ -# Qwen3.5 FP8 GB200 disaggregated MTP 7P1D DEP4/DEP16 point. - -name: "qwen3.5-fp8-gb200-mtp-8k1k-7p1d-dep4-dep16" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 6 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 7 - decode_nodes: 4 - prefill_workers: 7 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31001 - mem-fraction-static: 0.7 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31001 - - chunked-prefill-size: 4096 - context-length: 16384 - mem-fraction-static: 0.7 - max-mamba-cache-size: 2048 - max-running-requests: 2048 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 20 - num_warmup_mult: 2 - random_range_ratio: 0.8 - concurrencies: "1344" - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_4.yaml deleted file mode 100644 index 0f0469401c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_4.yaml +++ /dev/null @@ -1,186 +0,0 @@ -# Qwen3.5 FP8 GB200 disaggregated MTP 8P1D DEP4/DEP16 points. - -name: "qwen3.5-fp8-gb200-mtp-8k1k-8p1d-dep4-dep16" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 7 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 8 - decode_nodes: 4 - prefill_workers: 8 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31001 - mem-fraction-static: 0.7 - chunked-prefill-size: 98304 - max-prefill-tokens: 24576 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31001 - - chunked-prefill-size: 4096 - context-length: 16384 - mem-fraction-static: 0.7 - max-mamba-cache-size: 2048 - max-running-requests: 2048 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 20 - num_warmup_mult: 2 - random_range_ratio: 0.8 - concurrencies: "1920x2304" - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml deleted file mode 100644 index e4e023d385..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml +++ /dev/null @@ -1,190 +0,0 @@ -# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 1P1D: TP4 Prefill + TP4 Decode -# Pure tensor parallel, no expert parallel (STP) -# 8k1k sa-bench concurrency sweep on GB300 -# -# Values taken from ni_experiment_config of the -# sa-qwen-3.5-8k1k-fp4-baseline-low-latency study, row -# qwen3.5-1p_tp4x1d_tp4-aligned-ccsweep (CSV pareto export 2026-06-05). - -name: "gb300-fp4-qwen3.5_8k1k_lowlat_0" - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - version: "1.1.0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 2 - nginx_container: nginx - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - SGLANG_ENABLE_SPEC_V2: "1" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_ENABLE_JIT_DEEPGEMM: "true" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - FLASHINFER_DISABLE_VERSION_CHECK: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - SGLANG_ENABLE_SPEC_V2: "1" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_ENABLE_JIT_DEEPGEMM: "true" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - FLASHINFER_DISABLE_VERSION_CHECK: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" - model-path: "/model/" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - reasoning-parser: "qwen3" - tool-call-parser: "qwen3_coder" - - quantization: "modelopt_fp4" - fp4-gemm-backend: "flashinfer_cutlass" - kv-cache-dtype: "fp8_e4m3" - - mamba-scheduler-strategy: "no_buffer" - mamba-ssm-dtype: "bfloat16" - mamba-track-interval: 2048 - - attention-backend: "trtllm_mha" - mm-attention-backend: "triton_attn" - moe-runner-backend: "flashinfer_trtllm" - linear-attn-decode-backend: "flashinfer" - - disaggregation-mode: "prefill" - disable-radix-cache: true - - mem-fraction-static: 0.8 - context-length: 9236 - max-total-tokens: 128000 - max-running-requests: 128 - cuda-graph-max-bs: 4 - chunked-prefill-size: 32768 - max-prefill-tokens: 32768 - scheduler-recv-interval: 10 - stream-interval: 30 - load-balance-method: "round_robin" - page-size: 64 - watchdog-timeout: 1000000 - log-level: "info" - - decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" - model-path: "/model/" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - reasoning-parser: "qwen3" - tool-call-parser: "qwen3_coder" - - quantization: "modelopt_fp4" - fp4-gemm-backend: "flashinfer_cutlass" - kv-cache-dtype: "fp8_e4m3" - - mamba-scheduler-strategy: "no_buffer" - mamba-ssm-dtype: "bfloat16" - mamba-track-interval: 128 - - attention-backend: "trtllm_mha" - mm-attention-backend: "triton_attn" - moe-runner-backend: "flashinfer_trtllm" - linear-attn-decode-backend: "flashinfer" - - disaggregation-mode: "decode" - disable-radix-cache: true - - mem-fraction-static: 0.8 - context-length: 9236 - max-total-tokens: 1500000 - max-mamba-cache-size: 256 - max-running-requests: 128 - cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 - max-prefill-tokens: 32768 - scheduler-recv-interval: 10 - stream-interval: 30 - page-size: 64 - watchdog-timeout: 1000000 - decode-log-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1x4x8x16x32x64x256" - req_rate: "inf" - random_range_ratio: 0.8 - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - # 9401 is already bound by the cluster-level exporter on im-gb300 nodes; - # use a port outside that range. - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml deleted file mode 100644 index 29f5523057..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml +++ /dev/null @@ -1,190 +0,0 @@ -# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 5P1D wide-EP -# Prefill: 5 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) -# Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) -# Total: 36 GB300 GPUs (5*4 + 4*4); 8k1k concurrency 1024/2048/3072. -# -# Values taken from ni_experiment_config of pareto row -# qwen3.5-dep16-fia2a-tbo-cc1024x2048x3072-dynamo-tot-nixl -# (sa-qwen-3.5-8k1k-fp4-baseline-mid-pareto study). - -name: "gb300-fp4-qwen3.5_8k1k_maxtpt_0" - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - version: "1.1.0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 2 - nginx_container: nginx - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 5 - prefill_workers: 5 - decode_nodes: 4 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - SGLANG_ENABLE_SPEC_V2: "1" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_ENABLE_JIT_DEEPGEMM: "true" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" - FLASHINFER_DISABLE_VERSION_CHECK: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_ENABLE_NIXL: "1" - - decode_environment: - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - SGLANG_ENABLE_SPEC_V2: "1" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_ENABLE_JIT_DEEPGEMM: "true" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" - SGLANG_MOE_NVFP4_DISPATCH: "1" - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: "1" - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: "1" - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: "1" - FLASHINFER_DISABLE_VERSION_CHECK: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - SGLANG_ENABLE_NIXL: "1" - - sglang_config: - prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" - model-path: "/model/" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - disaggregation-transfer-backend: "nixl" - - mem-fraction-static: 0.8 - max-total-tokens: 128000 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - log-level: "info" - page-size: 64 - - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - linear-attn-decode-backend: "flashinfer" - - decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" - model-path: "/model/" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - enable-two-batch-overlap: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - disaggregation-transfer-backend: "nixl" - - chunked-prefill-size: 4096 - max-mamba-cache-size: 4096 - max-total-tokens: 2200000 - max-running-requests: 4096 - mem-fraction-static: 0.8 - watchdog-timeout: 1000000 - page-size: 64 - - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_cutedsl" - moe-a2a-backend: "flashinfer" - disable-shared-experts-fusion: true - linear-attn-decode-backend: "flashinfer" - - decode-log-interval: 50 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" - random_range_ratio: 0.8 - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml deleted file mode 100644 index 064463297b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml +++ /dev/null @@ -1,188 +0,0 @@ -# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 6P1D wide-EP -# Prefill: 6 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) -# Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) -# Total: 40 GB300 GPUs (6*4 + 4*4); 8k1k concurrency 5120. -# -# Values taken from ni_experiment_config of pareto row -# qwen3.5-6p_dep4x1d_dep16-fia2a-tbo-cc5120-dynamo-tot-mooncake -# (sa-qwen-3.5-8k1k-fp4-baseline-mid-pareto study). - -name: "gb300-fp4-qwen3.5_8k1k_maxtpt_1" - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - version: "1.1.0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 2 - nginx_container: nginx - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 6 - prefill_workers: 6 - decode_nodes: 4 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - SGLANG_ENABLE_SPEC_V2: "1" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_ENABLE_JIT_DEEPGEMM: "true" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" - FLASHINFER_DISABLE_VERSION_CHECK: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - SGLANG_ENABLE_SPEC_V2: "1" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_ENABLE_JIT_DEEPGEMM: "true" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" - SGLANG_MOE_NVFP4_DISPATCH: "1" - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: "1" - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: "1" - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: "1" - FLASHINFER_DISABLE_VERSION_CHECK: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" - model-path: "/model/" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - disaggregation-transfer-backend: "mooncake" - - mem-fraction-static: 0.8 - max-total-tokens: 128000 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - log-level: "info" - page-size: 64 - - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - linear-attn-decode-backend: "flashinfer" - - decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" - model-path: "/model/" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - enable-two-batch-overlap: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - disaggregation-transfer-backend: "mooncake" - - chunked-prefill-size: 5120 - max-mamba-cache-size: 5120 - max-total-tokens: 3200000 - max-running-requests: 5120 - mem-fraction-static: 0.8 - watchdog-timeout: 1000000 - page-size: 64 - - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_cutedsl" - moe-a2a-backend: "flashinfer" - disable-shared-experts-fusion: true - linear-attn-decode-backend: "flashinfer" - - decode-log-interval: 50 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "5120" - req_rate: "inf" - random_range_ratio: 0.8 - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml deleted file mode 100644 index 94f3f3509d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml +++ /dev/null @@ -1,188 +0,0 @@ -# Qwen3.5-397B-A17B-NVFP4-V2 Disaggregated 7P1D wide-EP -# Prefill: 7 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) -# Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) -# Total: 44 GB300 GPUs (7*4 + 4*4); 8k1k concurrency 5120. -# -# Values taken from ni_experiment_config of pareto row -# qwen3.5-7p_dep4x1d_dep16-fia2a-tbo-cc5120-dynamo-tot-mooncake -# (sa-qwen-3.5-8k1k-fp4-baseline-mid-pareto study). - -name: "gb300-fp4-qwen3.5_8k1k_maxtpt_2" - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - version: "1.1.0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 2 - nginx_container: nginx - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 7 - prefill_workers: 7 - decode_nodes: 4 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - SGLANG_ENABLE_SPEC_V2: "1" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_ENABLE_JIT_DEEPGEMM: "true" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" - FLASHINFER_DISABLE_VERSION_CHECK: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - SGLANG_ENABLE_SPEC_V2: "1" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_ENABLE_JIT_DEEPGEMM: "true" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" - SGLANG_MOE_NVFP4_DISPATCH: "1" - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: "1" - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: "1" - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: "1" - FLASHINFER_DISABLE_VERSION_CHECK: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" - model-path: "/model/" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - disaggregation-transfer-backend: "mooncake" - - mem-fraction-static: 0.8 - max-total-tokens: 128000 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - log-level: "info" - page-size: 64 - - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - linear-attn-decode-backend: "flashinfer" - - decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4-V2" - model-path: "/model/" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - enable-two-batch-overlap: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - disaggregation-transfer-backend: "mooncake" - - chunked-prefill-size: 5120 - max-mamba-cache-size: 5120 - max-total-tokens: 3200000 - max-running-requests: 5120 - mem-fraction-static: 0.8 - watchdog-timeout: 1000000 - page-size: 64 - - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_cutedsl" - moe-a2a-backend: "flashinfer" - disable-shared-experts-fusion: true - linear-attn-decode-backend: "flashinfer" - - decode-log-interval: 50 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "5120" - req_rate: "inf" - random_range_ratio: 0.8 - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml deleted file mode 100644 index 1ff01cb879..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml +++ /dev/null @@ -1,91 +0,0 @@ -name: agg-gb300-tp2-c1-mtp-hicache-jid2530006 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 2 -infra: - nats_max_payload_mb: 8 -frontend: - type: sglang -backend: - aggregated_environment: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 - type: sglang -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml deleted file mode 100644 index a3e1e47e50..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml +++ /dev/null @@ -1,91 +0,0 @@ -name: agg-gb300-tp2-c24-mtp-hicache-jid2530012 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 2 -infra: - nats_max_payload_mb: 8 -frontend: - type: sglang -backend: - aggregated_environment: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 - type: sglang -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml deleted file mode 100644 index 77324cb75d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml +++ /dev/null @@ -1,91 +0,0 @@ -name: agg-gb300-tp2-c32-mtp-hicache-jid2530013 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 2 -infra: - nats_max_payload_mb: 8 -frontend: - type: sglang -backend: - aggregated_environment: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 - type: sglang -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml deleted file mode 100644 index aca9321b4c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml +++ /dev/null @@ -1,91 +0,0 @@ -name: agg-gb300-tp2-c40-mtp-hicache-jid2530015 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 2 -infra: - nats_max_payload_mb: 8 -frontend: - type: sglang -backend: - aggregated_environment: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 - type: sglang -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831.yaml deleted file mode 100644 index bb42351a74..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831.yaml +++ /dev/null @@ -1,133 +0,0 @@ -name: agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831 -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 2 -infra: - nats_max_payload_mb: 8 -frontend: - type: sglang - enable_multiple_frontends: false -backend: - type: sglang - aggregated_environment: - SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/flashinfer-cache - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-c44-mrr1/sglang-cache - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 1 - moe-dense-tp-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false - moe-a2a-backend: none - load-balance-method: round_robin - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 8192 - mamba-max-states-per-path: 3 - mamba-ssm-dtype: bfloat16 - max-mamba-cache-size: 1536 - context-length: 262144 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: triton - disable-shared-experts-fusion: true - speculative-algorithm: NEXTN - speculative-num-steps: 6 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - speculative-moe-runner-backend: flashinfer_trtllm - speculative-moe-a2a-backend: none - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - prefill-decode-interval: 0 - mem-fraction-static: 0.85 - max-running-requests: 1 - pp-max-micro-batch-size: 1 - prefill-max-requests: 1 - stream-interval: 20 - decode-log-interval: 10 - watchdog-timeout: 1000000 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-size: 32 - enable-metrics: true - enable-linear-replayssm-spec: true - disable-attn-tp-gather: true - cuda-graph-max-bs-decode: 1 - cuda-graph-bs-decode: - - 1 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831.yaml deleted file mode 100644 index 5839754fb7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831.yaml +++ /dev/null @@ -1,134 +0,0 @@ -name: agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831 -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 2 -infra: - nats_max_payload_mb: 8 -frontend: - type: sglang - enable_multiple_frontends: false -backend: - type: sglang - aggregated_environment: - SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/flashinfer-cache - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2/sglang-cache - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 1 - moe-dense-tp-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false - moe-a2a-backend: none - load-balance-method: round_robin - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 8192 - mamba-max-states-per-path: 3 - mamba-ssm-dtype: bfloat16 - max-mamba-cache-size: 1536 - context-length: 262144 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: triton - disable-shared-experts-fusion: true - speculative-algorithm: NEXTN - speculative-num-steps: 6 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - speculative-moe-runner-backend: flashinfer_trtllm - speculative-moe-a2a-backend: none - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - prefill-decode-interval: 0 - mem-fraction-static: 0.85 - max-running-requests: 2 - pp-max-micro-batch-size: 2 - prefill-max-requests: 2 - cuda-graph-max-bs-decode: 2 - cuda-graph-bs-decode: - - 1 - - 2 - stream-interval: 20 - decode-log-interval: 10 - watchdog-timeout: 1000000 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-size: 128 - enable-metrics: true - enable-linear-replayssm-spec: true - disable-attn-tp-gather: true -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml deleted file mode 100644 index 9aed4b026f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml +++ /dev/null @@ -1,91 +0,0 @@ -name: agg-gb300-tp2-c48-mtp-hicache-jid2530017 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 2 -infra: - nats_max_payload_mb: 8 -frontend: - type: sglang -backend: - aggregated_environment: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 - type: sglang -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml deleted file mode 100644 index 17c2095906..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml +++ /dev/null @@ -1,91 +0,0 @@ -name: agg-gb300-tp2-c52-mtp-hicache-jid2527406 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 2 -infra: - nats_max_payload_mb: 8 -frontend: - type: sglang -backend: - aggregated_environment: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 - type: sglang -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml deleted file mode 100644 index d5a14ddf4d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml +++ /dev/null @@ -1,91 +0,0 @@ -name: agg-gb300-tp2-c64-mtp-hicache-jid2527410 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 2 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 2 -infra: - nats_max_payload_mb: 8 -frontend: - type: sglang -backend: - aggregated_environment: - NCCL_NVLS_ENABLE: '1' - PYTHONNOUSERSITE: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - SGL_ENABLE_JIT_DEEPGEMM: 'false' - TORCH_CUDA_ARCH_LIST: '10.0' - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 80 - max-running-requests: 72 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.75 - max-mamba-cache-size: 360 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 0.9 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_back - mamba-max-states-per-path: 1 - type: sglang -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'false' - TP: '2' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831.yaml deleted file mode 100644 index 56470bff4e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831.yaml +++ /dev/null @@ -1,133 +0,0 @@ -name: agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831 -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 -infra: - nats_max_payload_mb: 8 -frontend: - type: sglang - enable_multiple_frontends: false -backend: - type: sglang - aggregated_environment: - SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/c7-mrr1/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/c7-mrr1/flashinfer-cache - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/c7-mrr1/sglang-cache - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - data-parallel-size: 1 - expert-parallel-size: 1 - moe-dense-tp-size: 8 - enable-dp-attention: false - enable-dp-lm-head: false - moe-a2a-backend: none - load-balance-method: round_robin - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 8192 - mamba-max-states-per-path: 3 - mamba-ssm-dtype: bfloat16 - max-mamba-cache-size: 1536 - context-length: 262144 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - linear-attn-decode-backend: triton - disable-shared-experts-fusion: true - speculative-algorithm: NEXTN - speculative-num-steps: 6 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 7 - speculative-moe-runner-backend: flashinfer_cutedsl - speculative-moe-a2a-backend: none - chunked-prefill-size: 8192 - max-prefill-tokens: 8192 - mem-fraction-static: 0.85 - max-running-requests: 1 - pp-max-micro-batch-size: 1 - prefill-max-requests: 1 - cuda-graph-max-bs-decode: 1 - cuda-graph-bs-decode: - - 1 - disable-prefill-cuda-graph: true - stream-interval: 20 - decode-log-interval: 10 - watchdog-timeout: 1000000 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-size: 32 - enable-metrics: true - enable-linear-replayssm-spec: true - disable-attn-tp-gather: true -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - TP: '8' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml deleted file mode 100644 index 626f1e21b3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml +++ /dev/null @@ -1,206 +0,0 @@ -name: disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - decode_nodes: 1 - decode_workers: 1 - gpu_type: gb300 - gpus_per_decode: 2 - gpus_per_node: 2 - gpus_per_prefill: 2 - prefill_nodes: 1 - prefill_workers: 1 -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -backend: - decode_environment: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - prefill_environment: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - sglang_config: - decode: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 80 - max-running-requests: 80 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - decode-log-interval: 10 - mamba-max-states-per-path: -1 - prefill: - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 2 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - max-mamba-cache-size: 320 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - mamba-max-states-per-path: 1 - type: sglang -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml deleted file mode 100644 index bf0b6dd7e2..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml +++ /dev/null @@ -1,201 +0,0 @@ -name: disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - decode_nodes: 1 - decode_workers: 1 - gpu_type: gb300 - gpus_per_decode: 4 - gpus_per_node: 4 - gpus_per_prefill: 4 - prefill_nodes: 1 - prefill_workers: 1 -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -backend: - decode_environment: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - prefill_environment: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - sglang_config: - decode: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 160 - watchdog-timeout: 1000000 - decode-log-interval: 10 - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - type: sglang -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml deleted file mode 100644 index 1657ffb134..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml +++ /dev/null @@ -1,201 +0,0 @@ -name: disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - decode_nodes: 1 - decode_workers: 1 - gpu_type: gb300 - gpus_per_decode: 4 - gpus_per_node: 4 - gpus_per_prefill: 4 - prefill_nodes: 1 - prefill_workers: 1 -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -backend: - decode_environment: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - prefill_environment: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - sglang_config: - decode: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 64 - watchdog-timeout: 1000000 - decode-log-interval: 10 - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - type: sglang -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml deleted file mode 100644 index 67bfe4f61c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml +++ /dev/null @@ -1,201 +0,0 @@ -name: disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - decode_nodes: 1 - decode_workers: 1 - gpu_type: gb300 - gpus_per_decode: 4 - gpus_per_node: 4 - gpus_per_prefill: 4 - prefill_nodes: 1 - prefill_workers: 1 -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -backend: - decode_environment: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - prefill_environment: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - sglang_config: - decode: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 64 - watchdog-timeout: 1000000 - decode-log-interval: 10 - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - type: sglang -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml deleted file mode 100644 index 14012bd254..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml +++ /dev/null @@ -1,201 +0,0 @@ -name: disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - decode_nodes: 1 - decode_workers: 1 - gpu_type: gb300 - gpus_per_decode: 4 - gpus_per_node: 4 - gpus_per_prefill: 4 - prefill_nodes: 1 - prefill_workers: 1 -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -backend: - decode_environment: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - prefill_environment: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - sglang_config: - decode: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 80 - watchdog-timeout: 1000000 - decode-log-interval: 10 - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - type: sglang -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml deleted file mode 100644 index f33326871d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml +++ /dev/null @@ -1,201 +0,0 @@ -name: disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - decode_nodes: 1 - decode_workers: 1 - gpu_type: gb300 - gpus_per_decode: 4 - gpus_per_node: 4 - gpus_per_prefill: 4 - prefill_nodes: 1 - prefill_workers: 1 -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -backend: - decode_environment: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - prefill_environment: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - sglang_config: - decode: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 64 - watchdog-timeout: 1000000 - decode-log-interval: 10 - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - type: sglang -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml deleted file mode 100644 index 0584b59017..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml +++ /dev/null @@ -1,201 +0,0 @@ -name: disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409 -model: - path: qwen3.5-fp4 - container: dynamo-sglang - precision: fp4 -dynamo: - install: true - hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - decode_nodes: 1 - decode_workers: 1 - gpu_type: gb300 - gpus_per_decode: 4 - gpus_per_node: 4 - gpus_per_prefill: 4 - prefill_nodes: 1 - prefill_workers: 1 -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - nginx_session_affinity_header: X-Dynamo-Session-ID -backend: - decode_environment: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - prefill_environment: - FLASHINFER_DISABLE_VERSION_CHECK: '1' - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - MC_FORCE_MNNVL: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NO_COLOR: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - PYTHONUNBUFFERED: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token - sglang_config: - decode: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - decode-log-interval: 10 - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-cache-report: true - enable-metrics: true - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1100 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - type: sglang -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831.yaml deleted file mode 100644 index 777e4f91dd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831.yaml +++ /dev/null @@ -1,239 +0,0 @@ -name: disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831 -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 -dynamo: - install: true - hash: 61c37bc33c3e88da3facaa39fa2b5caf616740ba -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 2 - prefill_workers: 3 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 4 -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 1 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-replica-sync: true - router-queue-threshold: None - router-temperature: '1.0' -backend: - type: sglang - prefill_environment: - # The 0831 image contains the PP+spec runtime merged by SGLang #35758, - # but still carries the pre-merge PP+spec assert in validation_hook.py. - PYTHONOPTIMIZE: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-upstream-main/prefill-deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-upstream-main/prefill-flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '8192' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_PP_LAYER_PARTITION: 16,16,16,12 - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - SGLANG_FLASHINFER_AUTOTUNE_EXTEND: '1' - SGLANG_CACHE_DIR: /tmp/agentx-upstream-main/prefill-sglang-cache - decode_environment: - SGLANG_USE_SYMM_MEM_DP_SYNC: 'true' - SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '4' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-upstream-main/decode-deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-upstream-main/decode-flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_CACHE_DIR: /tmp/agentx-upstream-main/decode-sglang-cache - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - sglang_config: - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 1 - pipeline-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - enable-symm-mem: false - disable-overlap-schedule: true - enable-dynamic-chunking: false - mamba-ssm-dtype: bfloat16 - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 1048576 - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1536 - max-running-requests: 128 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - linear-attn-decode-backend: triton - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - watchdog-timeout: 1000000 - log-level: info - nccl-port: 29500 - scheduler-recv-interval: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - moe-dense-tp-size: 1 - disable-shared-experts-fusion: true - moe-a2a-backend: none - speculative-moe-a2a-backend: none - speculative-moe-runner-backend: flashinfer_trtllm - pp-async-batch-depth: 1 - disable-cuda-graph: true - decode: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - data-parallel-size: 4 - expert-parallel-size: 4 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: round_robin - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - linear-attn-decode-backend: triton - disable-shared-experts-fusion: true - moe-a2a-backend: flashinfer - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 5 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 6 - speculative-draft-model-quantization: nvfp4_online - speculative-moe-runner-backend: flashinfer_trtllm_routed - speculative-moe-a2a-backend: flashinfer - chunked-prefill-size: 4096 - mem-fraction-static: 0.7 - max-mamba-cache-size: 468 - max-running-requests: 320 - cuda-graph-max-bs: 80 - disaggregation-decode-extra-slots: 2 - stream-interval: 30 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 32 - decode-log-interval: 30 - watchdog-timeout: 1000000 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831.yaml deleted file mode 100644 index 821463150d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831.yaml +++ /dev/null @@ -1,239 +0,0 @@ -name: disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831 -model: - path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260831-bb5e6198 - precision: fp4 -dynamo: - install: true - hash: 61c37bc33c3e88da3facaa39fa2b5caf616740ba -slurm: - time_limit: '8:00:00' -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 4 - prefill_workers: 3 - decode_workers: 4 - gpus_per_prefill: 4 - gpus_per_decode: 4 -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 8 -frontend: - type: dynamo - nginx_session_affinity: true - enable_multiple_frontends: true - num_additional_frontends: 1 - env: - PIP_BREAK_SYSTEM_PACKAGES: '1' - args: - router-mode: kv - router-session-affinity-ttl-secs: '3600' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - router-replica-sync: true - router-queue-threshold: None - router-temperature: '1.0' -backend: - type: sglang - prefill_environment: - # The 0831 image contains the PP+spec runtime merged by SGLang #35758, - # but still carries the pre-merge PP+spec assert in validation_hook.py. - PYTHONOPTIMIZE: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '8192' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_PP_LAYER_PARTITION: 16,16,16,12 - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - SGLANG_FLASHINFER_AUTOTUNE_EXTEND: '1' - SGLANG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/prefill-sglang-cache - decode_environment: - SGLANG_USE_SYMM_MEM_DP_SYNC: 'true' - SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '4' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - PYTHONUNBUFFERED: '1' - PIP_BREAK_SYSTEM_PACKAGES: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/decode-deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-followup/c565-cutedsl-g3072/decode-flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - SGLANG_CACHE_DIR: /tmp/agentx-followup/c565-cutedsl-g3072/decode-sglang-cache - SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' - sglang_config: - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 1 - pipeline-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - enable-symm-mem: false - disable-overlap-schedule: true - enable-dynamic-chunking: false - mamba-ssm-dtype: bfloat16 - mamba-radix-cache-strategy: extra_buffer - mamba-track-interval: 1048576 - mamba-max-states-per-path: 3 - max-mamba-cache-size: 1536 - max-running-requests: 128 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - speculative-algorithm: NEXTN - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - linear-attn-decode-backend: triton - mem-fraction-static: 0.85 - max-prefill-tokens: 32768 - chunked-prefill-size: 32768 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 0.9 - watchdog-timeout: 1000000 - log-level: info - nccl-port: 29500 - scheduler-recv-interval: 1 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 - moe-dense-tp-size: 1 - disable-shared-experts-fusion: true - moe-a2a-backend: none - speculative-moe-a2a-backend: none - speculative-moe-runner-backend: flashinfer_trtllm - pp-async-batch-depth: 1 - disable-cuda-graph: true - decode: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - enable-metrics: true - enable-cache-report: true - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - data-parallel-size: 4 - expert-parallel-size: 4 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: round_robin - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - linear-attn-decode-backend: triton - disable-shared-experts-fusion: true - moe-a2a-backend: flashinfer - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 4 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 5 - speculative-draft-model-quantization: nvfp4_online - speculative-moe-runner-backend: flashinfer_trtllm_routed - speculative-moe-a2a-backend: flashinfer - chunked-prefill-size: 4096 - mem-fraction-static: 0.7 - max-mamba-cache-size: 468 - max-running-requests: 320 - cuda-graph-max-bs: 80 - disaggregation-decode-extra-slots: 2 - stream-interval: 30 - enable-linear-replayssm-spec: true - linear-replayssm-cache-len: 32 - decode-log-interval: 30 - watchdog-timeout: 1000000 - weight-loader-prefetch-checkpoints: true - weight-loader-prefetch-num-threads: 4 -sbatch_directives: - mem: '0' - cpus-per-task: '144' -srun_options: - mem: '0' - container-remap-root: '' -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-dep4-dep16.yaml deleted file mode 100644 index 8fdc4c2157..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-dep4-dep16.yaml +++ /dev/null @@ -1,167 +0,0 @@ -name: "qwen3.5-1p1d-dep4-dep16" - - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" - precision: "fp8" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 4096 - mem-fraction-static: 0.80 - max-mamba-cache-size: 2048 - max-running-requests: 2048 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "512x1024x2048" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-tp4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-tp4-tp4.yaml deleted file mode 100644 index 71b8c903a0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-tp4-tp4.yaml +++ /dev/null @@ -1,136 +0,0 @@ -name: "qwen3.5-1p1d-tp4-tp4" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" - precision: "fp8" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_HEALTH_CHECK_TIMEOUT: "3600" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 4096 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "prefill" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 4096 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "decode" - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 10 - num_warmup_mult: 1 - random_range_ratio: 0.8 - concurrencies: "1x2x4x8x16x32x64" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/2p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/2p1d-dep4-dep16.yaml deleted file mode 100644 index ef6b3f32fb..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/2p1d-dep4-dep16.yaml +++ /dev/null @@ -1,167 +0,0 @@ -name: "qwen3.5-2p1d-dep4-dep16" - - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 2 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" - precision: "fp8" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 4 - prefill_workers: 2 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 8192 - mem-fraction-static: 0.75 - max-mamba-cache-size: 4096 - max-running-requests: 4096 - cuda-graph-max-bs: 256 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "4096" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml deleted file mode 100644 index 359eab8a66..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml +++ /dev/null @@ -1,136 +0,0 @@ -name: "qwen3.5-1p1d-tp4-tp4" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" - precision: "fp8" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_HEALTH_CHECK_TIMEOUT: "3600" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 16384 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "prefill" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 16384 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "decode" - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "1x2x4x8x16x32x64x128" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - # 9401 is already bound by the cluster-level exporter on im-gb300 nodes; - # use a port outside that range. - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/4p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/4p1d-dep4-dep16.yaml deleted file mode 100644 index 7528260bb3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/4p1d-dep4-dep16.yaml +++ /dev/null @@ -1,172 +0,0 @@ -name: "qwen3.5-4p1d-dep4-dep16" - - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" - precision: "fp8" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 4 - prefill_workers: 4 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 16384 - mem-fraction-static: 0.80 - max-mamba-cache-size: 2048 - max-running-requests: 2048 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 20 - num_warmup_mult: 2 - random_range_ratio: 0.8 - concurrencies: "1024" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/8p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/8p1d-dep4-dep16.yaml deleted file mode 100644 index 61da0a4ad8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/8p1d-dep4-dep16.yaml +++ /dev/null @@ -1,160 +0,0 @@ -name: "qwen3.5-8p1d-dep4-dep16" - - -sbatch_directives: - mem: "0" - -infra: - etcd_nats_dedicated_node: true - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" - precision: "fp8" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 8 - decode_nodes: 4 - prefill_workers: 8 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 16384 - mem-fraction-static: 0.80 - max-mamba-cache-size: 2048 - max-running-requests: 2048 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "2048x4096" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml deleted file mode 100644 index 7478e636be..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml +++ /dev/null @@ -1,156 +0,0 @@ -# Qwen3.5 FP8 GB300 disaggregated MTP 1P1D TP4/TP4 configuration. - -name: "qwen3.5-fp8-gb300-mtp-8k1k-1p1d-tp4-tp4" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp8" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 16384 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "prefill" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - max-mamba-cache-size: 1024 - moe-runner-backend: "flashinfer_trtllm" - - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 16384 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "decode" - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "1x2x8" - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml deleted file mode 100644 index 25b6c2c924..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml +++ /dev/null @@ -1,170 +0,0 @@ -# Qwen3.5 FP8 GB300 disaggregated MTP 1P1D TEP8/TEP8 configuration. - -name: "qwen3.5-fp8-gb300-mtp-8k1k-1p1d-tep8-tep8" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp8" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 8 - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - max-total-tokens: 128000 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 9236 - cuda-graph-max-bs: 320 - scheduler-recv-interval: 10 - decode-log-interval: 50 - stream-interval: 50 - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "mooncake" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 8 - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - max-mamba-cache-size: 1024 - moe-runner-backend: "flashinfer_trtllm" - - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - max-total-tokens: 2200000 - chunked-prefill-size: 4096 - max-prefill-tokens: 16384 - context-length: 9236 - cuda-graph-max-bs: 320 - scheduler-recv-interval: 10 - decode-log-interval: 50 - stream-interval: 50 - disaggregation-mode: "decode" - disaggregation-transfer-backend: "mooncake" - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 20 - num_warmup_mult: 2 - random_range_ratio: 0.8 - concurrencies: "32x48x80" - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_0.yaml deleted file mode 100644 index 486ee3a5c3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_0.yaml +++ /dev/null @@ -1,186 +0,0 @@ -# Qwen3.5 FP8 GB300 disaggregated MTP 3P1D DEP4/DEP16 configuration. - -name: "qwen3.5-fp8-gb300-mtp-8k1k-3p1d-dep4-dep16" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 2 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp8" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 4 - prefill_workers: 3 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.7 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 16384 - mem-fraction-static: 0.7 - max-mamba-cache-size: 1024 - max-running-requests: 1024 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 20 - num_warmup_mult: 2 - random_range_ratio: 0.8 - concurrencies: "480" - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_1.yaml deleted file mode 100644 index 92ce213546..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_1.yaml +++ /dev/null @@ -1,186 +0,0 @@ -# Qwen3.5 FP8 GB300 disaggregated MTP 4P1D DEP4/DEP16 configuration. - -name: "qwen3.5-fp8-gb300-mtp-8k1k-4p1d-dep4-dep16" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp8" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 4 - prefill_workers: 4 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.7 - chunked-prefill-size: 98304 - max-prefill-tokens: 24576 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 16384 - mem-fraction-static: 0.7 - max-mamba-cache-size: 1024 - max-running-requests: 1024 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 20 - num_warmup_mult: 2 - random_range_ratio: 0.8 - concurrencies: "768" - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_2.yaml deleted file mode 100644 index d1263f74fe..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_2.yaml +++ /dev/null @@ -1,186 +0,0 @@ -# Qwen3.5 FP8 GB300 disaggregated MTP 6P1D DEP4/DEP16 configuration. - -name: "qwen3.5-fp8-gb300-mtp-8k1k-6p1d-dep4-dep16" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 5 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp8" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 6 - decode_nodes: 4 - prefill_workers: 6 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31001 - mem-fraction-static: 0.7 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31001 - - chunked-prefill-size: 4096 - context-length: 16384 - mem-fraction-static: 0.7 - max-mamba-cache-size: 1024 - max-running-requests: 1024 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 20 - num_warmup_mult: 2 - random_range_ratio: 0.8 - concurrencies: "1280" - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_3.yaml deleted file mode 100644 index d3c3a11daf..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_3.yaml +++ /dev/null @@ -1,186 +0,0 @@ -# Qwen3.5 FP8 GB300 disaggregated MTP 7P1D DEP4/DEP16 configuration. - -name: "qwen3.5-fp8-gb300-mtp-8k1k-7p1d-dep4-dep16" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 6 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp8" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 7 - decode_nodes: 4 - prefill_workers: 7 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31001 - mem-fraction-static: 0.7 - chunked-prefill-size: 65536 - max-prefill-tokens: 16384 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31001 - - chunked-prefill-size: 4096 - context-length: 16384 - mem-fraction-static: 0.7 - max-mamba-cache-size: 2048 - max-running-requests: 2048 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 20 - num_warmup_mult: 2 - random_range_ratio: 0.8 - concurrencies: "1344" - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_4.yaml deleted file mode 100644 index 5fc1b667c1..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_4.yaml +++ /dev/null @@ -1,186 +0,0 @@ -# Qwen3.5 FP8 GB300 disaggregated MTP 8P1D DEP4/DEP16 configuration. - -name: "qwen3.5-fp8-gb300-mtp-8k1k-8p1d-dep4-dep16" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 7 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - precision: "fp8" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 8 - decode_nodes: 4 - prefill_workers: 8 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - SGLANG_ENABLE_SPEC_V2: "1" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31001 - mem-fraction-static: 0.7 - chunked-prefill-size: 98304 - max-prefill-tokens: 24576 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - random-seed: 42 - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31001 - - chunked-prefill-size: 4096 - context-length: 16384 - mem-fraction-static: 0.7 - max-mamba-cache-size: 2048 - max-running-requests: 2048 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 20 - num_warmup_mult: 2 - random_range_ratio: 0.8 - concurrencies: "1920x2304" - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp.yaml deleted file mode 100644 index ad25648f41..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp.yaml +++ /dev/null @@ -1,191 +0,0 @@ -name: dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 8 - decode_workers: 1 - gpus_per_decode: 32 -backend: - type: trtllm - prefill_environment: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - decode_environment: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - trtllm_config: - prefill: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: null - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 256 - max_num_tokens: 16384 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 8 - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.7 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 4 - max_num_tokens: 16 - max_seq_len: 996595 - moe_config: - backend: MEGAMOE_DEEPGEMM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 32 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 32 - publish_events_and_metrics: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '388' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - orchestrator_placement: first_decode -benchmark: - client_placement: last_decode - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '388' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp.yaml deleted file mode 100644 index 0defd7e765..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp.yaml +++ /dev/null @@ -1,191 +0,0 @@ -name: dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 8 - decode_workers: 4 - gpus_per_decode: 8 -backend: - type: trtllm - prefill_environment: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - decode_environment: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - trtllm_config: - prefill: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: null - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 128 - max_num_tokens: 4096 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 4 - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.9 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 1 - max_num_tokens: 4 - max_seq_len: 996595 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 8 - publish_events_and_metrics: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '4' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - orchestrator_placement: first_decode -benchmark: - client_placement: last_decode - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '4' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp.yaml deleted file mode 100644 index 592cccdc98..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp.yaml +++ /dev/null @@ -1,191 +0,0 @@ -name: dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 6 - decode_workers: 6 - gpus_per_decode: 4 -backend: - type: trtllm - prefill_environment: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - decode_environment: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - trtllm_config: - prefill: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: null - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 128 - max_num_tokens: 4096 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 4 - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.9 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 4 - max_num_tokens: 16 - max_seq_len: 996595 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 4 - publish_events_and_metrics: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '24' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - orchestrator_placement: first_decode -benchmark: - client_placement: last_decode - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '24' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp.yaml deleted file mode 100644 index f7b17a3562..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp.yaml +++ /dev/null @@ -1,192 +0,0 @@ -name: dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 4 - prefill_workers: 2 - gpus_per_prefill: 8 - decode_nodes: 8 - decode_workers: 1 - gpus_per_decode: 32 -backend: - type: trtllm - prefill_environment: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - decode_environment: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - trtllm_config: - prefill: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: null - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 256 - max_num_tokens: 16384 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 8 - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.7 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 8 - max_num_tokens: 32 - max_seq_len: 996595 - moe_config: - backend: MEGAMOE_DEEPGEMM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 32 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 32 - publish_events_and_metrics: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '736' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - orchestrator_placement: first_decode -benchmark: - client_placement: last_decode - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '736' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp.yaml deleted file mode 100644 index 0c263389de..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp.yaml +++ /dev/null @@ -1,198 +0,0 @@ -name: dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 6 - prefill_workers: 3 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 -backend: - type: trtllm - prefill_environment: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - decode_environment: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - trtllm_config: - prefill: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: null - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 256 - max_num_tokens: 16384 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 8 - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 12 - - 16 - - 20 - - 24 - - 28 - - 32 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.7 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 32 - max_num_tokens: 128 - max_seq_len: 996595 - moe_config: - backend: MEGAMOE_DEEPGEMM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 16 - publish_events_and_metrics: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '1152' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - orchestrator_placement: first_decode -benchmark: - client_placement: last_decode - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '1152' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp.yaml deleted file mode 100644 index 3973be63a1..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp.yaml +++ /dev/null @@ -1,214 +0,0 @@ -name: dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp -model: - path: deepseek-ai/DeepSeek-V4-Pro - container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 10 - prefill_workers: 5 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 -backend: - type: trtllm - prefill_environment: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - CUDA_SCALE_LAUNCH_QUEUES: 4x - decode_environment: - OMP_NUM_THREADS: '1' - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.49' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: '1' - DYN_DEFAULT_THINKING_MODE: disabled - MIMALLOC_ARENA_RESERVE: '0' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - trtllm_config: - prefill: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: null - custom_tokenizer: deepseek_v4 - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 193273528320 - pool_ratio: - - 0.55 - - 0.22 - - 0.23 - tokens_per_block: 128 - block_reuse_config: - policy: per_conversation - max_num_turns: 5 - max_batch_size: 256 - max_num_tokens: 16384 - max_seq_len: 990016 - moe_config: - backend: TRTLLM - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 8 - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 12 - - 16 - - 20 - - 24 - - 28 - - 32 - - 36 - - 40 - - 44 - - 48 - - 52 - - 56 - - 60 - - 64 - - 68 - - 72 - - 76 - - 80 - - 84 - - 88 - - 92 - - 96 - enable_padding: true - custom_tokenizer: deepseek_v4 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 200000 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.7 - host_cache_size: 0 - tokens_per_block: 128 - max_batch_size: 96 - max_num_tokens: 384 - max_seq_len: 996595 - moe_config: - backend: MEGAMOE_DEEPGEMM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: deepseek_v4 - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 16 - publish_events_and_metrics: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '2626' - DURATION: '3600' - KV_OFFLOADING: dram - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - orchestrator_placement: first_decode -benchmark: - client_placement: last_decode - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - MODEL: deepseek-ai/DeepSeek-V4-Pro - MODEL_PREFIX: dsv4 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '2626' - DURATION: '3600' - KV_OFFLOADING: dram - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: DeepSeek-V4-Pro diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml deleted file mode 100644 index 096760c630..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml +++ /dev/null @@ -1,184 +0,0 @@ -name: dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: trtllm - prefill_environment: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - decode_environment: *server_environment - trtllm_config: - prefill: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: null - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - tensor_parallel_size: 4 - decode: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1, 2, 4, 5] - enable_padding: true - enable_attention_dp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - tokens_per_block: 64 - max_batch_size: 5 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - use_cute_dsl_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - stream_interval: 20 - tensor_parallel_size: 8 - publish_events_and_metrics: false - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - orchestrator_placement: head - -benchmark: - type: custom - client_dedicated_node: true - client_placement: head - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5.yaml deleted file mode 100644 index 4f47518c0d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5.yaml +++ /dev/null @@ -1,182 +0,0 @@ -name: dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: trtllm - prefill_environment: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - decode_environment: *server_environment - trtllm_config: - prefill: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: EQUAL_PROGRESS - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - tensor_parallel_size: 4 - decode: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1] - enable_padding: true - enable_attention_dp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - tokens_per_block: 64 - max_batch_size: 1 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 1 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - use_cute_dsl_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - stream_interval: 20 - tensor_parallel_size: 8 - publish_events_and_metrics: false - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - orchestrator_placement: head - -benchmark: - type: custom - client_dedicated_node: true - client_placement: head - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml deleted file mode 100644 index 17bfb46293..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml +++ /dev/null @@ -1,184 +0,0 @@ -name: dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 4 - decode_workers: 4 - gpus_per_decode: 4 - -backend: - type: trtllm - prefill_environment: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - decode_environment: *server_environment - trtllm_config: - prefill: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: null - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - tensor_parallel_size: 4 - decode: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1, 2] - enable_padding: true - enable_attention_dp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - tokens_per_block: 64 - max_batch_size: 2 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - use_cute_dsl_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - stream_interval: 20 - tensor_parallel_size: 4 - publish_events_and_metrics: false - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - orchestrator_placement: head - -benchmark: - type: custom - client_dedicated_node: true - client_placement: head - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml deleted file mode 100644 index 1a13af362c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml +++ /dev/null @@ -1,184 +0,0 @@ -name: dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 3 - prefill_workers: 3 - gpus_per_prefill: 4 - decode_nodes: 4 - decode_workers: 4 - gpus_per_decode: 4 - -backend: - type: trtllm - prefill_environment: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - decode_environment: *server_environment - trtllm_config: - prefill: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: null - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - tensor_parallel_size: 4 - decode: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1, 2, 4, 5] - enable_padding: true - enable_attention_dp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - tokens_per_block: 64 - max_batch_size: 5 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - use_cute_dsl_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 5 - stream_interval: 20 - tensor_parallel_size: 4 - publish_events_and_metrics: false - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - orchestrator_placement: head - -benchmark: - type: custom - client_dedicated_node: true - client_placement: head - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-4p1d-dep8-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-4p1d-dep8-c227-b16-mtp3.yaml deleted file mode 100644 index 9fc4217679..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-4p1d-dep8-c227-b16-mtp3.yaml +++ /dev/null @@ -1,185 +0,0 @@ -name: dynamo-disagg-gb300-4p1d-dep8-c227-b16-mtp3 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 4 - prefill_workers: 4 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: trtllm - prefill_environment: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "1.99" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - decode_environment: *server_environment - trtllm_config: - prefill: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: null - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 4 - decode: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1, 2, 4, 8, 16] - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.9 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 16 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 8 - publish_events_and_metrics: false - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - orchestrator_placement: head - -benchmark: - type: custom - client_dedicated_node: true - client_placement: head - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c260-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c260-b16-mtp3.yaml deleted file mode 100644 index fbf737057e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c260-b16-mtp3.yaml +++ /dev/null @@ -1,185 +0,0 @@ -name: dynamo-disagg-gb300-5p1d-dep16-c260-b16-mtp3 - -model: - path: nvidia/GLM-5.2-NVFP4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 - precision: fp4 - -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - request_plane: tcp - -health_check: - max_attempts: 270 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 5 - prefill_workers: 5 - gpus_per_prefill: 4 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -backend: - type: trtllm - prefill_environment: &server_environment - HF_HUB_OFFLINE: "1" - TRANSFORMERS_OFFLINE: "1" - TQDM_DISABLE: "1" - HF_HUB_DISABLE_PROGRESS_BARS: "1" - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: "1" - TRTLLM_WORKER_DISABLE_GC: "1" - TRTLLM_ENABLE_PDL: "1" - NCCL_GRAPH_MIXING_SUPPORT: "0" - MIMALLOC_PURGE_DELAY: "0" - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TRTLLM_FUSED_DSA_METADATA: "1" - TRTLLM_DSA_INDEXER_BF16: "1" - TRTLLM_SERVE_ENABLE_MSGSPEC: "1" - TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "1.99" - DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TOKENIZER: fastokens - DYN_PUBLISH_KV_EVENTS: "0" - decode_environment: *server_environment - trtllm_config: - prefill: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: null - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 - tokens_per_block: 64 - max_batch_size: 256 - max_num_tokens: 8192 - max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 4 - decode: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1, 2, 4, 8, 16] - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.9 - tokens_per_block: 64 - max_batch_size: 16 - max_num_tokens: 128 - max_seq_len: 1048576 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - pipeline_parallel_size: 1 - print_iter_log: true - enable_iter_perf_stats: true - return_perf_metrics: false - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - use_cute_dsl_paged_mqa_logits: true - use_cute_dsl_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 20 - tensor_parallel_size: 16 - publish_events_and_metrics: false - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_QUEUE_THRESHOLD: None - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TOKENIZER: fastokens - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - orchestrator_placement: head - -benchmark: - type: custom - client_dedicated_node: true - client_placement: head - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" - SERVED_MODEL_NAME: GLM-5.2-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml deleted file mode 100644 index 13da898bc4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml +++ /dev/null @@ -1,117 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep16_batch32_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 4 - gpus_per_decode: 16 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - trust_remote_code: true - max_batch_size: 32 - max_num_tokens: 128 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '615' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml deleted file mode 100644 index 73b2d10adc..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml +++ /dev/null @@ -1,121 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep16_batch64_eplb0_mtp1 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 4 - gpus_per_decode: 16 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 1 - decode: - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 128 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 1 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '1229' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml deleted file mode 100644 index 37912a792f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml +++ /dev/null @@ -1,115 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep32_batch16_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 8 - gpus_per_decode: 32 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 64 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '666' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml deleted file mode 100644 index 9d09244308..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep32_batch4_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 8 - gpus_per_decode: 32 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - trust_remote_code: true - max_batch_size: 4 - max_num_tokens: 16 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '180' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml deleted file mode 100644 index 5af56e0452..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml +++ /dev/null @@ -1,114 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep32_batch8_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 8 - gpus_per_decode: 32 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - trust_remote_code: true - max_batch_size: 8 - max_num_tokens: 32 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '308' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml deleted file mode 100644 index 6a40ed810c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml +++ /dev/null @@ -1,116 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch16_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 4 - decode_nodes: 8 - gpus_per_decode: 8 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 64 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - allreduce_strategy: MNNVL -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '92' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml deleted file mode 100644 index e2919b07c9..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml +++ /dev/null @@ -1,114 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch1_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 4 - decode_nodes: 8 - gpus_per_decode: 8 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 1 - max_num_tokens: 4 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - allreduce_strategy: MNNVL -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '8' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml deleted file mode 100644 index cb043e80a7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml +++ /dev/null @@ -1,114 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch4_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 4 - decode_nodes: 8 - gpus_per_decode: 8 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 4 - max_num_tokens: 16 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - allreduce_strategy: MNNVL -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '24' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml deleted file mode 100644 index 87fc3c9899..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml +++ /dev/null @@ -1,115 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch8_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 4 - decode_nodes: 8 - gpus_per_decode: 8 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 8 - max_num_tokens: 32 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - allreduce_strategy: MNNVL -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '40' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml deleted file mode 100644 index 3b76d83207..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen5tep4_batch1_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 5 - decode_nodes: 5 - gpus_per_decode: 4 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 1 - max_num_tokens: 4 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '10' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml deleted file mode 100644 index ae75aea604..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml +++ /dev/null @@ -1,129 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx2dep4_gen1dep16_batch128_eplb0_mtp1 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 2 - prefill_workers: 2 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 4 - gpus_per_decode: 16 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 1 - decode: - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - trust_remote_code: true - max_batch_size: 128 - max_num_tokens: 256 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 1 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '2253' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml deleted file mode 100644 index ed5867951f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx2dep4_gen1dep16_batch256_eplb256_mtp1 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 2 - prefill_workers: 2 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 4 - gpus_per_decode: 16 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 1 - decode: - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - trust_remote_code: true - max_batch_size: 256 - max_num_tokens: 512 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - - 136 - - 144 - - 152 - - 160 - - 168 - - 176 - - 184 - - 192 - - 200 - - 208 - - 216 - - 224 - - 232 - - 240 - - 248 - - 256 - moe_config: - backend: CUTEDSL - load_balancer: - layer_updates_per_iter: 1 - num_slots: 256 - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 1 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '4301' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml deleted file mode 100644 index d21c50f2f9..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml +++ /dev/null @@ -1,177 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx2dep4_gen1dep8_batch512_eplb0_mtp1 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 2 - prefill_workers: 2 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 2 - gpus_per_decode: 8 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 1 - decode: - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - trust_remote_code: true - max_batch_size: 512 - max_num_tokens: 1024 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - - 136 - - 144 - - 152 - - 160 - - 168 - - 176 - - 184 - - 192 - - 200 - - 208 - - 216 - - 224 - - 232 - - 240 - - 248 - - 256 - - 264 - - 272 - - 280 - - 288 - - 296 - - 304 - - 312 - - 320 - - 328 - - 336 - - 344 - - 352 - - 360 - - 368 - - 376 - - 384 - - 392 - - 400 - - 408 - - 416 - - 424 - - 432 - - 440 - - 448 - - 456 - - 464 - - 472 - - 480 - - 488 - - 496 - - 504 - - 512 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 1 -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '4301' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch16_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch16_eplb0_mtp0.yaml deleted file mode 100644 index d9b2a84884..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch16_eplb0_mtp0.yaml +++ /dev/null @@ -1,109 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep32_batch16_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 8 - gpus_per_decode: 32 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '666' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch32_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch32_eplb0_mtp0.yaml deleted file mode 100644 index 9d0cfb08c3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch32_eplb0_mtp0.yaml +++ /dev/null @@ -1,111 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep32_batch32_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 8 - gpus_per_decode: 32 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 32 - max_num_tokens: 32 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '1229' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep8_batch512_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep8_batch512_eplb0_mtp0.yaml deleted file mode 100644 index f5b2378ceb..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep8_batch512_eplb0_mtp0.yaml +++ /dev/null @@ -1,171 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen1dep8_batch512_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 2 - gpus_per_decode: 8 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 512 - max_num_tokens: 512 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - - 136 - - 144 - - 152 - - 160 - - 168 - - 176 - - 184 - - 192 - - 200 - - 208 - - 216 - - 224 - - 232 - - 240 - - 248 - - 256 - - 264 - - 272 - - 280 - - 288 - - 296 - - 304 - - 312 - - 320 - - 328 - - 336 - - 344 - - 352 - - 360 - - 368 - - 376 - - 384 - - 392 - - 400 - - 408 - - 416 - - 424 - - 432 - - 440 - - 448 - - 456 - - 464 - - 472 - - 480 - - 488 - - 496 - - 504 - - 512 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '4301' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch16_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch16_eplb0_mtp0.yaml deleted file mode 100644 index 8a4ab2820d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch16_eplb0_mtp0.yaml +++ /dev/null @@ -1,110 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch16_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 4 - decode_nodes: 8 - gpus_per_decode: 8 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - allreduce_strategy: MNNVL -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '84' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml deleted file mode 100644 index 9a46c38543..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml +++ /dev/null @@ -1,108 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch1_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 4 - decode_nodes: 8 - gpus_per_decode: 8 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 1 - max_num_tokens: 1 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - allreduce_strategy: MNNVL -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '4' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch32_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch32_eplb0_mtp0.yaml deleted file mode 100644 index b5cf5ee77e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch32_eplb0_mtp0.yaml +++ /dev/null @@ -1,112 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch32_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 4 - decode_nodes: 8 - gpus_per_decode: 8 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 32 - max_num_tokens: 32 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - allreduce_strategy: MNNVL -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '168' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml deleted file mode 100644 index 61473a1f7f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml +++ /dev/null @@ -1,108 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch4_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 4 - decode_nodes: 8 - gpus_per_decode: 8 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 4 - max_num_tokens: 4 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - allreduce_strategy: MNNVL -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '20' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch64_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch64_eplb0_mtp0.yaml deleted file mode 100644 index f99df622a3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch64_eplb0_mtp0.yaml +++ /dev/null @@ -1,116 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen4tep8_batch64_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 4 - decode_nodes: 8 - gpus_per_decode: 8 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 8 - moe_expert_parallel_size: 8 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 64 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - allreduce_strategy: MNNVL -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '284' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml deleted file mode 100644 index 7ef8e996f0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen5tep4_batch1_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 5 - decode_nodes: 5 - gpus_per_decode: 4 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 1 - max_num_tokens: 1 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '5' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml deleted file mode 100644 index e3df06c70d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx1dep4_gen5tep4_batch4_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 5 - decode_nodes: 5 - gpus_per_decode: 4 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 4 - max_num_tokens: 4 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '25' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch128_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch128_eplb0_mtp0.yaml deleted file mode 100644 index f3166be8c8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch128_eplb0_mtp0.yaml +++ /dev/null @@ -1,123 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx2dep4_gen1dep16_batch128_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 2 - prefill_workers: 2 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 4 - gpus_per_decode: 16 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 128 - max_num_tokens: 128 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.75 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '2151' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch256_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch256_eplb0_mtp0.yaml deleted file mode 100644 index 4c588afcf6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch256_eplb0_mtp0.yaml +++ /dev/null @@ -1,139 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx2dep4_gen1dep16_batch256_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 2 - prefill_workers: 2 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 4 - gpus_per_decode: 16 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 256 - max_num_tokens: 256 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - - 136 - - 144 - - 152 - - 160 - - 168 - - 176 - - 184 - - 192 - - 200 - - 208 - - 216 - - 224 - - 232 - - 240 - - 248 - - 256 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.75 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '4301' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch64_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch64_eplb0_mtp0.yaml deleted file mode 100644 index 4fa9b93291..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch64_eplb0_mtp0.yaml +++ /dev/null @@ -1,115 +0,0 @@ -name: glm5_nvfp4_ISL1K_OSL1K_ctx2dep4_gen1dep32_batch64_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 2 - prefill_workers: 2 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 8 - gpus_per_decode: 32 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16384 - max_seq_len: 1064 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 64 - max_seq_len: 2088 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 1024 - osl: 1024 - concurrencies: '2151' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml deleted file mode 100644 index 1d555a2504..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml +++ /dev/null @@ -1,129 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx12dep4_gen1dep16_batch128_eplb0_mtp1 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 12 - prefill_workers: 12 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 4 - gpus_per_decode: 16 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 1 - decode: - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - trust_remote_code: true - max_batch_size: 128 - max_num_tokens: 256 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 1 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '2253' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml deleted file mode 100644 index 9662511d2d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen1dep32_batch2_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 8 - gpus_per_decode: 32 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 8 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '90' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml deleted file mode 100644 index f82c0c2c6f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch1_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 5 - decode_nodes: 5 - gpus_per_decode: 4 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 1 - max_num_tokens: 4 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '5' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml deleted file mode 100644 index 99ff55044f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch2_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 5 - decode_nodes: 5 - gpus_per_decode: 4 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 8 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '15' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml deleted file mode 100644 index d72a5d0f2e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch4_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 5 - decode_nodes: 5 - gpus_per_decode: 4 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 4 - max_num_tokens: 16 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '30' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml deleted file mode 100644 index 5488068262..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx2dep4_gen1dep32_batch4_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 2 - prefill_workers: 2 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 8 - gpus_per_decode: 32 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - trust_remote_code: true - max_batch_size: 4 - max_num_tokens: 16 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '180' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml deleted file mode 100644 index 13905b0957..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml +++ /dev/null @@ -1,114 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx4dep4_gen1dep32_batch8_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 4 - prefill_workers: 4 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 8 - gpus_per_decode: 32 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - trust_remote_code: true - max_batch_size: 8 - max_num_tokens: 32 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '333' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml deleted file mode 100644 index 0d28018a20..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml +++ /dev/null @@ -1,117 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx6dep4_gen1dep16_batch32_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 6 - prefill_workers: 6 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 4 - gpus_per_decode: 16 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - trust_remote_code: true - max_batch_size: 32 - max_num_tokens: 128 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '615' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml deleted file mode 100644 index 3b1c6affcd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml +++ /dev/null @@ -1,115 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx7dep4_gen1dep32_batch16_eplb0_mtp3 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 7 - prefill_workers: 7 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 8 - gpus_per_decode: 32 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 - decode: - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 64 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 3 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '666' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml deleted file mode 100644 index 48a2401410..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml +++ /dev/null @@ -1,121 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx8dep4_gen1dep16_batch64_eplb0_mtp1 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 8 - prefill_workers: 8 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 4 - gpus_per_decode: 16 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 1 - decode: - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 128 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - speculative_config: - decoding_type: MTP - num_nextn_predict_layers: 1 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '1127' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml deleted file mode 100644 index e4ec638273..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml +++ /dev/null @@ -1,109 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch16_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 5 - decode_nodes: 5 - gpus_per_decode: 4 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '105' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml deleted file mode 100644 index 4ed13a020f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch1_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 5 - decode_nodes: 5 - gpus_per_decode: 4 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 1 - max_num_tokens: 1 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '5' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml deleted file mode 100644 index 8c8c759822..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch2_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 5 - decode_nodes: 5 - gpus_per_decode: 4 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 2 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '10' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml deleted file mode 100644 index 5019bb20ed..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch4_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 5 - decode_nodes: 5 - gpus_per_decode: 4 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 4 - max_num_tokens: 4 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '25' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml deleted file mode 100644 index 4f3971b6c7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml +++ /dev/null @@ -1,108 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx1dep4_gen5tep4_batch8_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_workers: 5 - decode_nodes: 5 - gpus_per_decode: 4 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 8 - max_num_tokens: 8 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '50' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml deleted file mode 100644 index 97b70e4833..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml +++ /dev/null @@ -1,108 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx2dep4_gen1dep32_batch8_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 2 - prefill_workers: 2 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 8 - gpus_per_decode: 32 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 8 - max_num_tokens: 8 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.75 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '308' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: false -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml deleted file mode 100644 index 2a48823f9d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml +++ /dev/null @@ -1,109 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx4dep4_gen1dep32_batch16_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 4 - prefill_workers: 4 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 8 - gpus_per_decode: 32 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 16 - max_num_tokens: 16 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.75 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '615' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb0_mtp0.yaml deleted file mode 100644 index a2f97d489e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb0_mtp0.yaml +++ /dev/null @@ -1,115 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx5dep4_gen1dep16_batch64_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 5 - prefill_workers: 5 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 4 - gpus_per_decode: 16 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 64 - max_num_tokens: 64 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '1127' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml deleted file mode 100644 index 461d14f5d3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml +++ /dev/null @@ -1,111 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx6dep4_gen1dep32_batch32_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 6 - prefill_workers: 6 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 8 - gpus_per_decode: 32 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 32 - max_num_tokens: 32 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.75 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '1229' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb0_mtp0.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb0_mtp0.yaml deleted file mode 100644 index ad55ac6056..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb0_mtp0.yaml +++ /dev/null @@ -1,123 +0,0 @@ -name: glm5_nvfp4_ISL8K_OSL1K_ctx9dep4_gen1dep16_batch128_eplb0_mtp0 -model: - path: nvidia/GLM-5-NVFP4 - container: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - precision: fp4 -resources: - gpu_type: gb200 - prefill_nodes: 9 - prefill_workers: 9 - gpus_per_prefill: 4 - decode_workers: 1 - decode_nodes: 4 - gpus_per_decode: 16 - gpus_per_node: 4 -backend: - type: trtllm - prefill_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - decode_environment: - ENROOT_ALLOW_DEV: 'yes' - MIMALLOC_PURGE_DELAY: '0' - NCCL_GRAPH_MIXING_SUPPORT: '0' - TLLM_LOG_LEVEL: INFO - TRTLLM_ENABLE_PDL: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: 'cuda_ipc,cuda_copy,sm,self,tcp' - trtllm_config: - prefill: - tensor_parallel_size: 4 - moe_expert_parallel_size: 4 - pipeline_parallel_size: 1 - enable_attention_dp: true - disable_overlap_scheduler: true - trust_remote_code: true - max_batch_size: 2 - max_num_tokens: 16640 - max_seq_len: 8232 - print_iter_log: true - cuda_graph_config: null - moe_config: - backend: CUTEDSL - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - decode: - tensor_parallel_size: 16 - moe_expert_parallel_size: 16 - pipeline_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - trust_remote_code: true - max_batch_size: 128 - max_num_tokens: 128 - max_seq_len: 9256 - print_iter_log: true - stream_interval: 100 - num_postprocess_workers: 4 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - cache_transceiver_config: - backend: UCX - max_tokens_in_buffer: 16384 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '2151' - req_rate: 'inf' - num_prompts_mult: 16 - use_chat_template: true -frontend: - type: dynamo - enable_multiple_frontends: true -health_check: - max_attempts: 360 - interval_seconds: 10 -dynamo: - install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p1d-dep4-dep8-c308-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p1d-dep4-dep8-c308-stp.yaml deleted file mode 100644 index 1cb6ab0de4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p1d-dep4-dep8-c308-stp.yaml +++ /dev/null @@ -1,128 +0,0 @@ -name: disagg-B200-1p1d-dep4-dep8-c308-stp -model: - path: kimik2.5-fp4 - container: dynamo-trtllm - precision: fp4 -identity: - model: - repo: nvidia/Kimi-K2.5-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 -slurm: - time_limit: 04:00:00 -dynamo: - install: true - hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 - request_plane: tcp -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 2 - max_num_tokens: 16386 - max_seq_len: 8232 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 4 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.92 - max_batch_size: 32 - max_num_tokens: 32 - max_seq_len: 9256 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 308 - req_rate: inf - num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p4d-dep4-tep8-c24-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p4d-dep4-tep8-c24-stp.yaml deleted file mode 100644 index 49a71471de..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p4d-dep4-tep8-c24-stp.yaml +++ /dev/null @@ -1,124 +0,0 @@ -name: disagg-B200-1p4d-dep4-tep8-c24-stp -model: - path: kimik2.5-fp4 - container: dynamo-trtllm - precision: fp4 -identity: - model: - repo: nvidia/Kimi-K2.5-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 -slurm: - time_limit: 04:00:00 -dynamo: - install: true - hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 - request_plane: tcp -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 4 - decode_workers: 4 - gpus_per_decode: 8 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 2 - max_num_tokens: 16386 - max_seq_len: 8232 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 4 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 4 - max_num_tokens: 4 - max_seq_len: 9256 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 24 - req_rate: inf - num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p4d-dep4-tep8-c4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p4d-dep4-tep8-c4-stp.yaml deleted file mode 100644 index cabec31b18..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p4d-dep4-tep8-c4-stp.yaml +++ /dev/null @@ -1,124 +0,0 @@ -name: disagg-B200-1p4d-dep4-tep8-c4-stp -model: - path: kimik2.5-fp4 - container: dynamo-trtllm - precision: fp4 -identity: - model: - repo: nvidia/Kimi-K2.5-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 -slurm: - time_limit: 04:00:00 -dynamo: - install: true - hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 - request_plane: tcp -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 4 - decode_workers: 4 - gpus_per_decode: 8 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 2 - max_num_tokens: 16386 - max_seq_len: 8232 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 4 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 1 - max_num_tokens: 1 - max_seq_len: 9256 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 4 - req_rate: inf - num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c115-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c115-stp.yaml deleted file mode 100644 index 298279a010..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c115-stp.yaml +++ /dev/null @@ -1,126 +0,0 @@ -name: disagg-B200-1p5d-dep4-tep4-c115-stp -model: - path: kimik2.5-fp4 - container: dynamo-trtllm - precision: fp4 -identity: - model: - repo: nvidia/Kimi-K2.5-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 -slurm: - time_limit: 04:00:00 -dynamo: - install: true - hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 - request_plane: tcp -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 3 - decode_workers: 5 - gpus_per_decode: 4 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 2 - max_num_tokens: 16386 - max_seq_len: 8232 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 4 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - enable_padding: true - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 16 - max_num_tokens: 16 - max_seq_len: 9256 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 4 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 115 - req_rate: inf - num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c195-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c195-stp.yaml deleted file mode 100644 index f31a7da80e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c195-stp.yaml +++ /dev/null @@ -1,128 +0,0 @@ -name: disagg-B200-1p5d-dep4-tep4-c195-stp -model: - path: kimik2.5-fp4 - container: dynamo-trtllm - precision: fp4 -identity: - model: - repo: nvidia/Kimi-K2.5-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 -slurm: - time_limit: 04:00:00 -dynamo: - install: true - hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 - request_plane: tcp -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 3 - decode_workers: 5 - gpus_per_decode: 4 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 2 - max_num_tokens: 16386 - max_seq_len: 8232 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 4 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - enable_padding: true - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 32 - max_num_tokens: 32 - max_seq_len: 9256 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 4 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 195 - req_rate: inf - num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c30-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c30-stp.yaml deleted file mode 100644 index 4c4c833192..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c30-stp.yaml +++ /dev/null @@ -1,124 +0,0 @@ -name: disagg-B200-1p5d-dep4-tep4-c30-stp -model: - path: kimik2.5-fp4 - container: dynamo-trtllm - precision: fp4 -identity: - model: - repo: nvidia/Kimi-K2.5-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 -slurm: - time_limit: 04:00:00 -dynamo: - install: true - hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 - request_plane: tcp -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 3 - decode_workers: 5 - gpus_per_decode: 4 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 2 - max_num_tokens: 16386 - max_seq_len: 8232 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 4 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 4 - max_num_tokens: 4 - max_seq_len: 9256 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 4 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 30 - req_rate: inf - num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c5-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c5-stp.yaml deleted file mode 100644 index 619c1bb709..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c5-stp.yaml +++ /dev/null @@ -1,124 +0,0 @@ -name: disagg-B200-1p5d-dep4-tep4-c5-stp -model: - path: kimik2.5-fp4 - container: dynamo-trtllm - precision: fp4 -identity: - model: - repo: nvidia/Kimi-K2.5-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 -slurm: - time_limit: 04:00:00 -dynamo: - install: true - hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 - request_plane: tcp -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 3 - decode_workers: 5 - gpus_per_decode: 4 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 2 - max_num_tokens: 16386 - max_seq_len: 8232 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 4 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 1 - max_num_tokens: 1 - max_seq_len: 9256 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 4 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 5 - req_rate: inf - num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c60-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c60-stp.yaml deleted file mode 100644 index 1a8b4f2988..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c60-stp.yaml +++ /dev/null @@ -1,125 +0,0 @@ -name: disagg-B200-1p5d-dep4-tep4-c60-stp -model: - path: kimik2.5-fp4 - container: dynamo-trtllm - precision: fp4 -identity: - model: - repo: nvidia/Kimi-K2.5-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 -slurm: - time_limit: 04:00:00 -dynamo: - install: true - hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 - request_plane: tcp -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 3 - decode_workers: 5 - gpus_per_decode: 4 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 2 - max_num_tokens: 16386 - max_seq_len: 8232 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 4 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - enable_padding: true - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 8 - max_num_tokens: 8 - max_seq_len: 9256 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 4 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 60 - req_rate: inf - num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-2p1d-dep4-dep8-c615-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-2p1d-dep4-dep8-c615-stp.yaml deleted file mode 100644 index abcce625e7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-2p1d-dep4-dep8-c615-stp.yaml +++ /dev/null @@ -1,132 +0,0 @@ -name: disagg-B200-2p1d-dep4-dep8-c615-stp -model: - path: kimik2.5-fp4 - container: dynamo-trtllm - precision: fp4 -identity: - model: - repo: nvidia/Kimi-K2.5-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 -slurm: - time_limit: 04:00:00 -dynamo: - install: true - hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 - request_plane: tcp -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - prefill_workers: 2 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 2 - max_num_tokens: 16386 - max_seq_len: 8232 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 4 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.92 - max_batch_size: 64 - max_num_tokens: 64 - max_seq_len: 9256 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 615 - req_rate: inf - num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-3p1d-dep4-dep8-c1127-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-3p1d-dep4-dep8-c1127-stp.yaml deleted file mode 100644 index 316a88f23a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-3p1d-dep4-dep8-c1127-stp.yaml +++ /dev/null @@ -1,140 +0,0 @@ -name: disagg-B200-3p1d-dep4-dep8-c1127-stp -model: - path: kimik2.5-fp4 - container: dynamo-trtllm - precision: fp4 -identity: - model: - repo: nvidia/Kimi-K2.5-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 -slurm: - time_limit: 04:00:00 -dynamo: - install: true - hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 - request_plane: tcp -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 2 - prefill_workers: 3 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 2 - max_num_tokens: 16386 - max_seq_len: 8232 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 4 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.92 - max_batch_size: 128 - max_num_tokens: 128 - max_seq_len: 9256 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 1127 - req_rate: inf - num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-4p1d-dep4-dep8-c2151-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-4p1d-dep4-dep8-c2151-stp.yaml deleted file mode 100644 index 893a725cae..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-4p1d-dep4-dep8-c2151-stp.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: disagg-B200-4p1d-dep4-dep8-c2151-stp -model: - path: kimik2.5-fp4 - container: dynamo-trtllm - precision: fp4 -identity: - model: - repo: nvidia/Kimi-K2.5-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 -slurm: - time_limit: 04:00:00 -dynamo: - install: true - hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 - request_plane: tcp -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 2 - prefill_workers: 4 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 8 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 2 - max_num_tokens: 16386 - max_seq_len: 8232 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 4 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: DEFAULT - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 68736 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - - 136 - - 144 - - 152 - - 160 - - 168 - - 176 - - 184 - - 192 - - 200 - - 208 - - 216 - - 224 - - 232 - - 240 - - 248 - - 256 - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.92 - max_batch_size: 256 - max_num_tokens: 256 - max_seq_len: 9256 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 2151 - req_rate: inf - num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c10-b1-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c10-b1-eagle3.yaml deleted file mode 100644 index b59b35ec37..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c10-b1-eagle3.yaml +++ /dev/null @@ -1,149 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=10 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# Forced acceptance is NOT set here: runners/inject_synthetic_acceptance.py injects TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -name: dynamo-agg-gb200-tp4-c10-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 -backend: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - aggregated_environment: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - trtllm_config: - aggregated: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 10 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 - publish_events_and_metrics: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '10' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c15-b1-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c15-b1-eagle3.yaml deleted file mode 100644 index 096537ca1e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c15-b1-eagle3.yaml +++ /dev/null @@ -1,154 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=15 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# Forced acceptance is NOT set here: runners/inject_synthetic_acceptance.py injects TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -name: dynamo-agg-gb200-tp4-c15-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 -backend: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - aggregated_environment: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - trtllm_config: - aggregated: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 15 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 11 - - 12 - - 13 - - 14 - - 15 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 - publish_events_and_metrics: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '15' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c20-b1-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c20-b1-eagle3.yaml deleted file mode 100644 index 0183937d81..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c20-b1-eagle3.yaml +++ /dev/null @@ -1,159 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=20 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# Forced acceptance is NOT set here: runners/inject_synthetic_acceptance.py injects TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -name: dynamo-agg-gb200-tp4-c20-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 -backend: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - aggregated_environment: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - trtllm_config: - aggregated: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 20 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 11 - - 12 - - 13 - - 14 - - 15 - - 16 - - 17 - - 18 - - 19 - - 20 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 - publish_events_and_metrics: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '20' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c25-b1-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c25-b1-eagle3.yaml deleted file mode 100644 index afe297af0c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c25-b1-eagle3.yaml +++ /dev/null @@ -1,159 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=25 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# Forced acceptance is NOT set here: runners/inject_synthetic_acceptance.py injects TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -name: dynamo-agg-gb200-tp4-c25-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 -backend: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - aggregated_environment: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - trtllm_config: - aggregated: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 25 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 11 - - 12 - - 13 - - 14 - - 15 - - 17 - - 19 - - 21 - - 23 - - 25 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 - publish_events_and_metrics: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '25' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c30-b1-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c30-b1-eagle3.yaml deleted file mode 100644 index e13ad34d7c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c30-b1-eagle3.yaml +++ /dev/null @@ -1,159 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=30 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# Forced acceptance is NOT set here: runners/inject_synthetic_acceptance.py injects TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -name: dynamo-agg-gb200-tp4-c30-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 -backend: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - aggregated_environment: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - trtllm_config: - aggregated: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 30 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 11 - - 12 - - 14 - - 16 - - 18 - - 20 - - 22 - - 24 - - 27 - - 30 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 - publish_events_and_metrics: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '30' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c40-b1-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c40-b1-eagle3.yaml deleted file mode 100644 index da4340027d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c40-b1-eagle3.yaml +++ /dev/null @@ -1,163 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=40 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# Forced acceptance is NOT set here: runners/inject_synthetic_acceptance.py injects TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -name: dynamo-agg-gb200-tp4-c40-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 -backend: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - aggregated_environment: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - trtllm_config: - aggregated: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 40 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 - - 7 - - 8 - - 9 - - 10 - - 14 - - 16 - - 18 - - 20 - - 22 - - 24 - - 26 - - 28 - - 30 - - 32 - - 34 - - 36 - - 38 - - 40 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 - publish_events_and_metrics: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '40' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c5-b1-eagle3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c5-b1-eagle3.yaml deleted file mode 100644 index e526cfe83e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c5-b1-eagle3.yaml +++ /dev/null @@ -1,144 +0,0 @@ -# MiniMax-M3 NVFP4, TensorRT-LLM aggregated TP4 + EAGLE3-GQA (3 draft tokens), one GB200 node, Dynamo frontend. -# Engine config is the max_batch_size=5 member of the TP4 recipe set. -# Host KV cache is set to 128 GiB per rank for the four-rank layout. -# Forced acceptance is NOT set here: runners/inject_synthetic_acceptance.py injects TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS -# (= AL - 1 = 1.78 for the committed golden AL 2.78) for throughput runs and leaves eval-only runs on real acceptance. -name: dynamo-agg-gb200-tp4-c5-b1-eagle3 -model: - path: minimax-m3-nvfp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 - precision: fp4 -dynamo: - install: true - wheel: 1.4.0.dev20260807 - request_plane: tcp -health_check: - max_attempts: 270 - interval_seconds: 10 -resources: - gpu_type: gb200 - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 -backend: - type: trtllm - numa_memory_bind: true - served_model_name: nvidia/MiniMax-M3-NVFP4 - aggregated_environment: - HF_HUB_OFFLINE: '1' - TRANSFORMERS_OFFLINE: '1' - HF_HUB_CACHE: /hf_hub_cache - TQDM_DISABLE: '1' - HF_HUB_DISABLE_PROGRESS_BARS: '1' - TLLM_LOG_LEVEL: INFO - TLLM_PROFILE_LOG_RANKS: all - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - PYTHONNOUSERSITE: '1' - TRTLLM_SERVE_ENABLE_MSGSPEC: '1' - TRTLLM_TORCH_COMPILE_CONTEXT_ONLY: '1' - DYN_PUBLISH_KV_EVENTS: '0' - trtllm_config: - aggregated: - max_seq_len: 1048576 - max_num_tokens: 16384 - max_batch_size: 5 - cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 3 - - 4 - - 5 - torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: - - 1 - - 512 - - 1024 - - 2048 - enable_userbuffers: true - max_num_streams: 3 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true - kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: 137438953472 - speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: Inferact/MiniMax-M3-EAGLE3-GQA - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - enable_chunked_prefill: true - enable_autotuner: true - trust_remote_code: true - stream_interval: 20 - print_iter_log: true - num_postprocess_workers: 8 - enable_attention_dp: false - tensor_parallel_size: 4 - publish_events_and_metrics: false -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_ROUTER_TEMPERATURE: '0' - DYN_TOKENIZER_CACHE: '1' - DYN_TOKENIZER_CACHE_BYTES: '8000000000' - DYN_TCP_REQUEST_TIMEOUT: '30' - DYN_LOG: warn - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' - args: - router-mode: kv - no-kv-events: true -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' - SERVED_MODEL_NAME: nvidia/MiniMax-M3-NVFP4 - MAX_MODEL_LEN: '1048576' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - OPENAI_API_KEY: EMPTY - KV_OFFLOADING: dram - KV_OFFLOAD_BACKEND: native - KV_OFFLOAD_BACKEND_METADATA: '{"name":"native"}' - TOTAL_CPU_DRAM_GB: '550' - MODEL: nvidia/MiniMax-M3-NVFP4 - MODEL_PREFIX: minimaxm3 - FRAMEWORK: dynamo-trt - PRECISION: fp4 - CONC: '5' - DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx11_gen1_dep8_batch128_eplb0_mtp3_1229.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx11_gen1_dep8_batch128_eplb0_mtp3_1229.yaml deleted file mode 100644 index 2c31d7e28b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx11_gen1_dep8_batch128_eplb0_mtp3_1229.yaml +++ /dev/null @@ -1,169 +0,0 @@ -name: disagg-gb300-11p1d-dep1-dep8-c1229-mtp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 3 - prefill_workers: 11 - gpus_per_prefill: 1 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 -backend: - type: trtllm - sequential_node_start: 2 - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - transceiver_runtime: PYTHON - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - max_batch_size: 2 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 12 - - 16 - - 20 - - 24 - - 28 - - 32 - - 36 - - 40 - - 44 - - 48 - - 52 - - 56 - - 60 - - 64 - - 68 - - 72 - - 76 - - 80 - - 84 - - 88 - - 92 - - 96 - - 100 - - 104 - - 108 - - 112 - - 116 - - 120 - - 124 - - 128 - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - max_batch_size: 128 - max_num_tokens: 512 - max_seq_len: 9472 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 1229 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx17_gen2_dep8_batch64_eplb0_mtp3_1126.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx17_gen2_dep8_batch64_eplb0_mtp3_1126.yaml deleted file mode 100644 index 4cc846e1bf..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx17_gen2_dep8_batch64_eplb0_mtp3_1126.yaml +++ /dev/null @@ -1,151 +0,0 @@ -name: disagg-gb300-17p2d-dep1-dep8-c1126-mtp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 5 - prefill_workers: 17 - gpus_per_prefill: 1 - decode_nodes: 4 - decode_workers: 2 - gpus_per_decode: 8 -backend: - type: trtllm - sequential_node_start: 2 - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - max_batch_size: 2 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 12 - - 16 - - 20 - - 24 - - 28 - - 32 - - 36 - - 40 - - 44 - - 48 - - 52 - - 56 - - 60 - - 64 - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - max_batch_size: 64 - max_num_tokens: 256 - max_seq_len: 9472 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 1126 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen2_tep8_batch8_eplb0_mtp3_20.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen2_tep8_batch8_eplb0_mtp3_20.yaml deleted file mode 100644 index b595a5ff87..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen2_tep8_batch8_eplb0_mtp3_20.yaml +++ /dev/null @@ -1,134 +0,0 @@ -name: disagg-gb300-1p2d-dep1-tep8-c20-mtp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 1 - decode_nodes: 4 - decode_workers: 2 - gpus_per_decode: 8 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - max_batch_size: 32 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - enable_padding: true - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 8 - max_num_tokens: 32 - max_seq_len: 9472 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 20 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_12.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_12.yaml deleted file mode 100644 index f13e5392f4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_12.yaml +++ /dev/null @@ -1,133 +0,0 @@ -name: disagg-gb300-1p4d-dep1-tep8-c12-mtp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 1 - decode_nodes: 8 - decode_workers: 4 - gpus_per_decode: 8 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - max_batch_size: 32 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 1 - max_num_tokens: 4 - max_seq_len: 9472 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 12 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen4_tep8_batch2_eplb0_mtp3_8.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen4_tep8_batch2_eplb0_mtp3_8.yaml deleted file mode 100644 index 58ece185ab..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen4_tep8_batch2_eplb0_mtp3_8.yaml +++ /dev/null @@ -1,135 +0,0 @@ -name: disagg-gb300-1p4d-dep1-tep8-c8-mtp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 1 - decode_nodes: 8 - decode_workers: 4 - gpus_per_decode: 8 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - max_batch_size: 32 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 2 - max_num_tokens: 8 - max_seq_len: 9472 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 8 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx24_gen1_dep16_batch512_eplb0_mtp0_8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx24_gen1_dep16_batch512_eplb0_mtp0_8192.yaml deleted file mode 100644 index 5b3e044150..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx24_gen1_dep16_batch512_eplb0_mtp0_8192.yaml +++ /dev/null @@ -1,194 +0,0 @@ -name: disagg-gb300-24p1d-dep1-dep16-c8192-stp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 6 - prefill_workers: 24 - gpus_per_prefill: 1 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 -backend: - type: trtllm - sequential_node_start: 2 - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - max_batch_size: 2 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - - 136 - - 144 - - 152 - - 160 - - 168 - - 176 - - 184 - - 192 - - 200 - - 208 - - 216 - - 224 - - 232 - - 240 - - 248 - - 256 - - 264 - - 272 - - 280 - - 288 - - 296 - - 304 - - 312 - - 320 - - 328 - - 336 - - 344 - - 352 - - 360 - - 368 - - 376 - - 384 - - 392 - - 400 - - 408 - - 416 - - 424 - - 432 - - 440 - - 448 - - 456 - - 464 - - 472 - - 480 - - 488 - - 496 - - 504 - - 512 - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.75 - max_batch_size: 512 - max_num_tokens: 512 - max_seq_len: 9472 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 16 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 8192 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx26_gen1_dep16_batch256_eplb0_mtp2_4301.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx26_gen1_dep16_batch256_eplb0_mtp2_4301.yaml deleted file mode 100644 index e6915c5ef7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx26_gen1_dep16_batch256_eplb0_mtp2_4301.yaml +++ /dev/null @@ -1,201 +0,0 @@ -name: disagg-gb300-26p1d-dep1-dep16-c4301-mtp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 7 - prefill_workers: 26 - gpus_per_prefill: 1 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 -backend: - type: trtllm - sequential_node_start: 2 - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - transceiver_runtime: PYTHON - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - max_batch_size: 2 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 2 - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 12 - - 16 - - 20 - - 24 - - 28 - - 32 - - 36 - - 40 - - 44 - - 48 - - 52 - - 56 - - 60 - - 64 - - 68 - - 72 - - 76 - - 80 - - 84 - - 88 - - 92 - - 96 - - 100 - - 104 - - 108 - - 112 - - 116 - - 120 - - 124 - - 128 - - 132 - - 136 - - 140 - - 144 - - 148 - - 152 - - 156 - - 160 - - 164 - - 168 - - 172 - - 176 - - 180 - - 184 - - 188 - - 192 - - 196 - - 200 - - 204 - - 208 - - 212 - - 216 - - 220 - - 224 - - 228 - - 232 - - 236 - - 240 - - 244 - - 248 - - 252 - - 256 - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.75 - max_batch_size: 256 - max_num_tokens: 768 - max_seq_len: 9472 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 2 - stream_interval: 100 - tensor_parallel_size: 16 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 4301 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx4_gen3_tep8_batch32_eplb0_mtp3_126.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx4_gen3_tep8_batch32_eplb0_mtp3_126.yaml deleted file mode 100644 index c10ba86c45..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx4_gen3_tep8_batch32_eplb0_mtp3_126.yaml +++ /dev/null @@ -1,141 +0,0 @@ -name: disagg-gb300-4p3d-dep1-tep8-c126-mtp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 4 - gpus_per_prefill: 1 - decode_nodes: 6 - decode_workers: 3 - gpus_per_decode: 8 -backend: - type: trtllm - sequential_node_start: 2 - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - max_batch_size: 32 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 12 - - 16 - - 20 - - 24 - - 28 - - 32 - enable_padding: true - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 32 - max_num_tokens: 128 - max_seq_len: 9472 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 126 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx5_gen1_dep16_batch8_eplb0_mtp3_154.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx5_gen1_dep16_batch8_eplb0_mtp3_154.yaml deleted file mode 100644 index 8299757f52..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx5_gen1_dep16_batch8_eplb0_mtp3_154.yaml +++ /dev/null @@ -1,139 +0,0 @@ -name: disagg-gb300-5p1d-dep1-dep16-c154-mtp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 5 - gpus_per_prefill: 1 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 -backend: - type: trtllm - sequential_node_start: 2 - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - transceiver_runtime: PYTHON - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - max_batch_size: 2 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.75 - max_batch_size: 8 - max_num_tokens: 32 - max_seq_len: 9472 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 100 - tensor_parallel_size: 16 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 154 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx7_gen1_dep16_batch32_eplb0_mtp3_666.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx7_gen1_dep16_batch32_eplb0_mtp3_666.yaml deleted file mode 100644 index ac83aa5d78..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx7_gen1_dep16_batch32_eplb0_mtp3_666.yaml +++ /dev/null @@ -1,139 +0,0 @@ -name: disagg-gb300-7p1d-dep2-dep16-c666-mtp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 4 - prefill_workers: 7 - gpus_per_prefill: 2 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 -backend: - type: trtllm - sequential_node_start: 2 - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - max_batch_size: 32 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 2 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - tensor_parallel_size: 2 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 12 - - 16 - - 20 - - 24 - - 28 - - 32 - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.75 - max_batch_size: 32 - max_num_tokens: 128 - max_seq_len: 9472 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - speculative_config: - decoding_type: MTP - max_draft_len: 3 - stream_interval: 100 - tensor_parallel_size: 16 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 666 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx10_gen1_dep8_batch256_eplb0_mtp0_2150.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx10_gen1_dep8_batch256_eplb0_mtp0_2150.yaml deleted file mode 100644 index 856dcfa539..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx10_gen1_dep8_batch256_eplb0_mtp0_2150.yaml +++ /dev/null @@ -1,162 +0,0 @@ -name: disagg-gb300-10p1d-dep1-dep8-c2150-stp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 3 - prefill_workers: 10 - gpus_per_prefill: 1 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 -backend: - type: trtllm - sequential_node_start: 2 - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - max_batch_size: 2 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - - 136 - - 144 - - 152 - - 160 - - 168 - - 176 - - 184 - - 192 - - 200 - - 208 - - 216 - - 224 - - 232 - - 240 - - 248 - - 256 - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.8 - max_batch_size: 256 - max_num_tokens: 256 - max_seq_len: 9472 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 2150 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx11_gen1_dep16_batch64_eplb0_mtp0_1076.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx11_gen1_dep16_batch64_eplb0_mtp0_1076.yaml deleted file mode 100644 index eb81e30ff9..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx11_gen1_dep16_batch64_eplb0_mtp0_1076.yaml +++ /dev/null @@ -1,140 +0,0 @@ -name: disagg-gb300-11p1d-dep1-dep16-c1076-stp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 3 - prefill_workers: 11 - gpus_per_prefill: 1 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 -backend: - type: trtllm - sequential_node_start: 2 - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - transceiver_runtime: PYTHON - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - max_batch_size: 2 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.75 - max_batch_size: 64 - max_num_tokens: 64 - max_seq_len: 9472 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 16 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 1076 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx16_gen1_dep16_batch128_eplb0_mtp0_2253.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx16_gen1_dep16_batch128_eplb0_mtp0_2253.yaml deleted file mode 100644 index 5034c39d1e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx16_gen1_dep16_batch128_eplb0_mtp0_2253.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: disagg-gb300-16p1d-dep1-dep16-c2253-stp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 4 - prefill_workers: 16 - gpus_per_prefill: 1 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 -backend: - type: trtllm - sequential_node_start: 2 - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - transceiver_runtime: PYTHON - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - max_batch_size: 2 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.75 - max_batch_size: 128 - max_num_tokens: 128 - max_seq_len: 9472 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 16 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 2253 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen2_tep8_batch16_eplb0_mtp0_42.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen2_tep8_batch16_eplb0_mtp0_42.yaml deleted file mode 100644 index 084010e55d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen2_tep8_batch16_eplb0_mtp0_42.yaml +++ /dev/null @@ -1,129 +0,0 @@ -name: disagg-gb300-1p2d-dep1-tep8-c42-stp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 1 - decode_nodes: 4 - decode_workers: 2 - gpus_per_decode: 8 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - max_batch_size: 32 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - enable_padding: true - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 16 - max_num_tokens: 16 - max_seq_len: 9472 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 42 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_8.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_8.yaml deleted file mode 100644 index 4317340be6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_8.yaml +++ /dev/null @@ -1,125 +0,0 @@ -name: disagg-gb300-1p4d-dep2-tep8-c8-stp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 2 - decode_nodes: 8 - decode_workers: 4 - gpus_per_decode: 8 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - max_batch_size: 32 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 2 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 2 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 1 - max_num_tokens: 1 - max_seq_len: 9472 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 8 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml deleted file mode 100644 index 1972efbff4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml +++ /dev/null @@ -1,127 +0,0 @@ -name: disagg-gb300-1p4d-dep1-tep8-c24-stp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 1 - decode_nodes: 8 - decode_workers: 4 - gpus_per_decode: 8 -backend: - type: trtllm - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - max_batch_size: 32 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 4 - max_num_tokens: 4 - max_seq_len: 9472 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 24 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx24_gen1_dep16_batch512_eplb0_mtp0_8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx24_gen1_dep16_batch512_eplb0_mtp0_8192.yaml deleted file mode 100644 index 5b3e044150..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx24_gen1_dep16_batch512_eplb0_mtp0_8192.yaml +++ /dev/null @@ -1,194 +0,0 @@ -name: disagg-gb300-24p1d-dep1-dep16-c8192-stp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 6 - prefill_workers: 24 - gpus_per_prefill: 1 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 -backend: - type: trtllm - sequential_node_start: 2 - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - max_batch_size: 2 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - - 136 - - 144 - - 152 - - 160 - - 168 - - 176 - - 184 - - 192 - - 200 - - 208 - - 216 - - 224 - - 232 - - 240 - - 248 - - 256 - - 264 - - 272 - - 280 - - 288 - - 296 - - 304 - - 312 - - 320 - - 328 - - 336 - - 344 - - 352 - - 360 - - 368 - - 376 - - 384 - - 392 - - 400 - - 408 - - 416 - - 424 - - 432 - - 440 - - 448 - - 456 - - 464 - - 472 - - 480 - - 488 - - 496 - - 504 - - 512 - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.75 - max_batch_size: 512 - max_num_tokens: 512 - max_seq_len: 9472 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 16 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 8192 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx4_gen1_dep16_batch16_eplb0_mtp0_282.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx4_gen1_dep16_batch16_eplb0_mtp0_282.yaml deleted file mode 100644 index 6fa4cf9e14..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx4_gen1_dep16_batch16_eplb0_mtp0_282.yaml +++ /dev/null @@ -1,132 +0,0 @@ -name: disagg-gb300-4p1d-dep1-dep16-c282-stp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 4 - gpus_per_prefill: 1 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 -backend: - type: trtllm - sequential_node_start: 2 - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - max_batch_size: 2 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.75 - max_batch_size: 16 - max_num_tokens: 16 - max_seq_len: 9472 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 16 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 282 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx4_gen3_tep8_batch64_eplb0_mtp0_210.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx4_gen3_tep8_batch64_eplb0_mtp0_210.yaml deleted file mode 100644 index 3635f56335..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx4_gen3_tep8_batch64_eplb0_mtp0_210.yaml +++ /dev/null @@ -1,138 +0,0 @@ -name: disagg-gb300-4p3d-dep1-tep8-c210-stp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 4 - gpus_per_prefill: 1 - decode_nodes: 6 - decode_workers: 3 - gpus_per_decode: 8 -backend: - type: trtllm - sequential_node_start: 2 - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - max_batch_size: 32 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: UCX - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - enable_padding: true - enable_attention_dp: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.9 - max_batch_size: 64 - max_num_tokens: 64 - max_seq_len: 9472 - moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 210 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx7_gen1_dep16_batch32_eplb0_mtp0_563.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx7_gen1_dep16_batch32_eplb0_mtp0_563.yaml deleted file mode 100644 index d477032c59..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx7_gen1_dep16_batch32_eplb0_mtp0_563.yaml +++ /dev/null @@ -1,134 +0,0 @@ -name: disagg-gb300-7p1d-dep1-dep16-c563-stp -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - wheel: 1.4.0.dev20260807 - -identity: - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 7 - gpus_per_prefill: 1 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 -backend: - type: trtllm - sequential_node_start: 2 - prefill_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - CTX_LOAD_STAGGER_S: '180' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - decode_environment: - TLLM_LOG_LEVEL: INFO - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRTLLM_ENABLE_PDL: '1' - NCCL_GRAPH_MIXING_SUPPORT: '0' - MIMALLOC_PURGE_DELAY: '0' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - trtllm_config: - prefill: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: null - disable_overlap_scheduler: true - enable_attention_dp: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.7 - max_batch_size: 2 - max_num_tokens: 16896 - max_seq_len: 8448 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - tensor_parallel_size: 1 - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_timeout_ms: 600000 - max_tokens_in_buffer: 16384 - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - enable_padding: true - enable_attention_dp: true - enable_lm_head_tp_in_adp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.75 - max_batch_size: 32 - max_num_tokens: 32 - max_seq_len: 9472 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - stream_interval: 100 - tensor_parallel_size: 16 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: true - env: - ETCD_LEASE_TTL: '120' - DYN_TOKENIZER: "fastokens" - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: - - 563 - req_rate: inf - num_prompts_mult: 16 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml deleted file mode 100644 index f6d6be9311..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml +++ /dev/null @@ -1,219 +0,0 @@ -name: disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 1 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 2 -backend: - type: trtllm - publish_events_and_metrics: false - prefill_environment: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - decode_environment: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - trtllm_config: - prefill: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: null - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 8192 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 1 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - enable_padding: true - enable_attention_dp: false - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.85 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 8 - max_num_tokens: 56 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 2 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 6 - stream_interval: 20 - tensor_parallel_size: 2 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml deleted file mode 100644 index 69e689ebd4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml +++ /dev/null @@ -1,216 +0,0 @@ -name: disagg-gb300-1p7d-dep4-tep8-c7-b1-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 14 - decode_workers: 7 - gpus_per_decode: 8 -backend: - type: trtllm - publish_events_and_metrics: false - prefill_environment: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '4.04' - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - decode_environment: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '4.04' - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - trtllm_config: - prefill: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: null - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 8192 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 4 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - enable_attention_dp: false - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.85 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 1 - max_num_tokens: 8 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 7 - stream_interval: 20 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml deleted file mode 100644 index 6b84385dc7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml +++ /dev/null @@ -1,218 +0,0 @@ -name: disagg-gb300-2p2d-dep1-tep2-c52-b4-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 2 - gpus_per_prefill: 1 - decode_nodes: 1 - decode_workers: 2 - gpus_per_decode: 2 -backend: - type: trtllm - publish_events_and_metrics: false - prefill_environment: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - decode_environment: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - trtllm_config: - prefill: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: null - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 8192 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 1 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 1 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - enable_padding: true - enable_attention_dp: false - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.85 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 4 - max_num_tokens: 28 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 2 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 6 - stream_interval: 20 - tensor_parallel_size: 2 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml deleted file mode 100644 index 7dcc8a4a03..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml +++ /dev/null @@ -1,233 +0,0 @@ -name: disagg-gb300-2p3d-tep2-tep8-c96-b128-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 2 - gpus_per_prefill: 2 - decode_nodes: 6 - decode_workers: 3 - gpus_per_decode: 8 -backend: - type: trtllm - publish_events_and_metrics: false - prefill_environment: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - decode_environment: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - trtllm_config: - prefill: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: null - disable_overlap_scheduler: false - enable_attention_dp: false - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 16384 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 2 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 2 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - - 16384 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - - 40 - - 48 - - 56 - - 64 - - 72 - - 80 - - 88 - - 96 - - 104 - - 112 - - 120 - - 128 - enable_padding: true - enable_attention_dp: false - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: false - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.85 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 128 - max_num_tokens: 896 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 6 - stream_interval: 20 - tensor_parallel_size: 8 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml deleted file mode 100644 index d3b3354531..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml +++ /dev/null @@ -1,217 +0,0 @@ -name: disagg-gb300-3p1d-dep4-dep16-c565-b8-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 3 - prefill_workers: 3 - gpus_per_prefill: 4 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 -backend: - type: trtllm - publish_events_and_metrics: false - prefill_environment: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - decode_environment: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - trtllm_config: - prefill: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: null - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 8192 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 4 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - enable_padding: true - enable_attention_dp: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.85 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 8 - max_num_tokens: 56 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 16 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 6 - stream_interval: 20 - tensor_parallel_size: 16 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml deleted file mode 100644 index a6ace6277b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml +++ /dev/null @@ -1,220 +0,0 @@ -name: disagg-gb300-3p2d-dep4-dep4-c704-b32-mtp-kvoffload -model: - path: qwen3.5-fp4 - container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - precision: fp4 - -dynamo: - install: true - hash: 2cbbdc863c336c2cfffa2a3668e2a6661801b259 - -identity: - model: - repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 - frameworks: - tensorrt_llm: 1.3.0rc24 - -slurm: - time_limit: 04:00:00 -health_check: - max_attempts: 540 - interval_seconds: 10 -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 3 - prefill_workers: 3 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 2 - gpus_per_decode: 4 -backend: - type: trtllm - publish_events_and_metrics: false - prefill_environment: - CUDA_SCALE_LAUNCH_QUEUES: 4x - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - decode_environment: - MIMALLOC_ARENA_RESERVE: '0' - MIMALLOC_PURGE_DELAY: '' - NCCL_GRAPH_MIXING_SUPPORT: '0' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TLLM_LOG_LEVEL: INFO - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' - TRTLLM_ENABLE_PDL: '1' - TRTLLM_PINNED_WEIGHT_STAGING: '1' - TRTLLM_SERVER_DISABLE_GC: '1' - TRTLLM_WORKER_DISABLE_GC: '1' - UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp - DYN_ENGINE_CONV_AFFINITY: "1" - DYN_TRTLLM_SERVED_MODEL_NAME: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 - trtllm_config: - prefill: - attention_dp_config: - kv_cache_routing_conversation_affinity: true - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: null - disable_overlap_scheduler: false - enable_attention_dp: true - enable_chunked_prefill: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - kv_cache_config: - block_reuse_config: - max_num_turns: 3 - policy: per_conversation - dtype: fp8 - enable_block_reuse: true - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - mamba_state_config: - additional_snapshot_offsets_from_end: - - 2 - periodic_snapshot_interval: 0 - pool_ratio: - - 0.8 - - 0.2 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 64 - max_num_tokens: 8192 - max_seq_len: 262144 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_serve_frontends: 8 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - tensor_parallel_size: 4 - torch_compile_config: - capture_num_tokens: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 - - 256 - - 512 - - 1024 - - 2048 - - 4096 - - 8192 - enable_fullgraph: true - enable_piecewise_cuda_graph: true - trust_remote_code: true - decode: - cache_transceiver_config: - backend: NIXL - kv_transfer_sender_future_timeout_ms: 20 - kv_transfer_timeout_ms: 600000 - transceiver_runtime: PYTHON - cuda_graph_config: - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 24 - - 32 - enable_padding: true - enable_attention_dp: true - enable_iter_perf_stats: true - enable_iter_req_stats: false - enable_lm_head_tp_in_adp: true - kv_cache_config: - avg_seq_len: 102150 - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - host_cache_size: 137438953472 - tokens_per_block: 64 - use_kv_cache_manager_v2: true - max_batch_size: 32 - max_num_tokens: 224 - max_seq_len: 262148 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - moe_expert_parallel_size: 4 - num_postprocess_workers: 4 - num_serve_frontends: 4 - nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: false - scheduler_config: - capacity_scheduler_policy: GUARANTEED_NO_EVICT - speculative_config: - decoding_type: MTP - max_draft_len: 6 - stream_interval: 20 - tensor_parallel_size: 4 - trust_remote_code: true -frontend: - type: dynamo - enable_multiple_frontends: false - env: - ETCD_LEASE_TTL: "120" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - args: - router-mode: kv - router-session-affinity-ttl-secs: '14400' - active-decode-blocks-threshold: None - active-prefill-tokens-threshold: None - active-prefill-tokens-threshold-frac: None - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: '8000' - IS_MULTINODE: 'true' - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-gb200-low-latency-mtp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-gb200-low-latency-mtp2.yaml deleted file mode 100644 index ffc8e4ea1a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-gb200-low-latency-mtp2.yaml +++ /dev/null @@ -1,86 +0,0 @@ -name: "svf-vllm-agg-gb200-low-latency-mtp2" - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.1-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - aggregated_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - vllm_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - attention-config: '{"use_fp4_indexer_cache":true}' - tokenizer-mode: deepseek_v4 - max-model-len: 9280 - max-num-seqs: 4 - max-num-batched-tokens: 8192 - max-cudagraph-capture-size: 4 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.9 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.20.1-ubuntu2404" - frameworks: - dynamo: "1.2.0.dev20260426" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml deleted file mode 100644 index 72e6dd1b16..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "svf-vllm-disagg-b200-1p1d-dep8-dep8-mtp" - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -health_check: - max_attempts: 90 - interval_seconds: 10 -sbatch_directives: - segment: "1" -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - TORCH_SYMMMEM: "NVSHMEM" - NVSHMEM_DISABLE_CUDA_VMM: "1" - NVSHMEM_SYMMETRIC_SIZE: "16g" - NVSHMEM_INFO: "1" - NVSHMEM_DEBUG: "INFO" - NCCL_NVLS_ENABLE: "1" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - TORCH_SYMMMEM: "NVSHMEM" - NVSHMEM_DISABLE_CUDA_VMM: "1" - NVSHMEM_SYMMETRIC_SIZE: "16g" - NCCL_NVLS_ENABLE: "1" - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - enable-expert-parallel: true - enable-ep-weight-filter: true - enforce-eager: true - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - attention-config: '{"use_fp4_indexer_cache":true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - max-model-len: 9280 - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: false - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - enable-expert-parallel: true - enable-ep-weight-filter: true - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - attention-config: '{"use_fp4_indexer_cache":true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "512" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml deleted file mode 100644 index dc4e403072..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml +++ /dev/null @@ -1,128 +0,0 @@ -name: "svf-vllm-disagg-b200-1p1d-dep8-tp8-mtp" - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -sbatch_directives: - segment: "1" -health_check: - max_attempts: 90 - interval_seconds: 10 -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - TORCH_SYMMMEM: "NVSHMEM" - NVSHMEM_DISABLE_CUDA_VMM: "1" - NVSHMEM_SYMMETRIC_SIZE: "16g" - NVSHMEM_INFO: "1" - NVSHMEM_DEBUG: "INFO" - NCCL_NVLS_ENABLE: "1" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enforce-eager: true - max-model-len: 9280 - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - attention-config: '{"use_fp4_indexer_cache":true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - # numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - max-model-len: 16384 - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - max-num-seqs: 256 - max-cudagraph-capture-size: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1x8x16" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml deleted file mode 100644 index 1bebd09a27..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml +++ /dev/null @@ -1,144 +0,0 @@ -name: "svf-vllm-disagg-b200-1p2d-dep8-dep8-mtp" - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -sbatch_directives: - segment: "1" -health_check: - max_attempts: 90 - interval_seconds: 10 -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - TORCH_SYMMMEM: "NVSHMEM" - NVSHMEM_DISABLE_CUDA_VMM: "1" - NVSHMEM_SYMMETRIC_SIZE: "16g" - NVSHMEM_INFO: "1" - NVSHMEM_DEBUG: "INFO" - NCCL_NVLS_ENABLE: "1" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - TORCH_SYMMMEM: "NVSHMEM" - NVSHMEM_DISABLE_CUDA_VMM: "1" - NVSHMEM_SYMMETRIC_SIZE: "16g" - NCCL_NVLS_ENABLE: "1" - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - enable-expert-parallel: true - enable-ep-weight-filter: true - enforce-eager: true - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - attention-config: '{"use_fp4_indexer_cache":true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - max-model-len: 9280 - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - # numa-bind: true - tokenizer-mode: deepseek_v4 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - enable-expert-parallel: true - enable-ep-weight-filter: true - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - attention-config: '{"use_fp4_indexer_cache":true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - max-model-len: 9280 - max-num-seqs: 128 - max-cudagraph-capture-size: 128 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "128x256" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml deleted file mode 100644 index 2c9e9e2feb..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "svf-vllm-disagg-b200-2p1d-dep8-dep8-mtp" - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -sbatch_directives: - segment: "1" -health_check: - max_attempts: 90 - interval_seconds: 10 -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 2 - decode_nodes: 1 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - TORCH_SYMMMEM: "NVSHMEM" - NVSHMEM_DISABLE_CUDA_VMM: "1" - NVSHMEM_SYMMETRIC_SIZE: "16g" - NVSHMEM_INFO: "1" - NVSHMEM_DEBUG: "INFO" - NCCL_NVLS_ENABLE: "1" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - TORCH_SYMMMEM: "NVSHMEM" - NVSHMEM_DISABLE_CUDA_VMM: "1" - NVSHMEM_SYMMETRIC_SIZE: "16g" - NCCL_NVLS_ENABLE: "1" - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - enable-expert-parallel: true - enable-ep-weight-filter: true - enforce-eager: true - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - attention-config: '{"use_fp4_indexer_cache":true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - max-model-len: 9280 - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - # numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - enable-expert-parallel: true - enable-ep-weight-filter: true - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - attention-config: '{"use_fp4_indexer_cache":true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1024" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml deleted file mode 100644 index 0b7841266e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "svf-vllm-disagg-b200-3p1d-dep8-dep8-mtp" - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -health_check: - max_attempts: 90 - interval_seconds: 10 -sbatch_directives: - segment: "1" -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 3 - decode_nodes: 1 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - TORCH_SYMMMEM: "NVSHMEM" - NVSHMEM_DISABLE_CUDA_VMM: "1" - NVSHMEM_SYMMETRIC_SIZE: "16g" - NVSHMEM_INFO: "1" - NVSHMEM_DEBUG: "INFO" - NCCL_NVLS_ENABLE: "1" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - TORCH_SYMMMEM: "NVSHMEM" - NVSHMEM_DISABLE_CUDA_VMM: "1" - NVSHMEM_SYMMETRIC_SIZE: "16g" - NCCL_NVLS_ENABLE: "1" - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - enable-expert-parallel: true - enable-ep-weight-filter: true - enforce-eager: true - speculative-config: '{"method":"mtp","num_speculative_tokens":1}' - attention-config: '{"use_fp4_indexer_cache":true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - max-model-len: 9280 - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - # numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - enable-expert-parallel: true - enable-ep-weight-filter: true - speculative-config: '{"method":"mtp","num_speculative_tokens":1}' - attention-config: '{"use_fp4_indexer_cache":true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c1.yaml deleted file mode 100644 index 26805d4efd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c1.yaml +++ /dev/null @@ -1,127 +0,0 @@ -name: "svf-vllm-disagg-b200-low-latency-c1" -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" -dynamo: - install: true - wheel: "1.2.0.dev20260426" -setup_script: vllm-container-deps.sh -slurm: - time_limit: "8:00:00" -health_check: - interval_seconds: 10 - max_attempts: 1440 -sbatch_directives: - segment: "1" -resources: - decode_nodes: 1 - decode_workers: 1 - gpu_type: b200 - gpus_per_decode: 8 - gpus_per_node: 8 - gpus_per_prefill: 8 - prefill_nodes: 1 - prefill_workers: 1 -infra: - etcd_nats_dedicated_node: true -frontend: - enable_multiple_frontends: false - type: dynamo -backend: - set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids - connector: null - decode_environment: - NCCL_CUMEM_ENABLE: '1' - TILELANG_CLEANUP_TEMP_FILES: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_SERVER_DEV_MODE: '1' - prefill_environment: - NCCL_CUMEM_ENABLE: '1' - TILELANG_CLEANUP_TEMP_FILES: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' - VLLM_SERVER_DEV_MODE: '1' - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' - type: vllm - vllm_config: - decode: - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - enable-sleep-mode: true - gpu-memory-utilization: 0.9 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - max-cudagraph-capture-size: 256 - max-model-len: 9280 - max-num-batched-tokens: 256 - max-num-seqs: 256 - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - pipeline-parallel-size: 1 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - stream-interval: 50 - tensor-parallel-size: 8 - tokenizer-mode: deepseek_v4 - trust-remote-code: true - prefill: - block-size: 256 - data-parallel-rpc-port: 13345 - data-parallel-size: 8 - enable-expert-parallel: true - enable-sleep-mode: true - enforce-eager: true - gpu-memory-utilization: 0.8 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - max-model-len: 9280 - max-num-batched-tokens: 32768 - max-num-seqs: 16 - no-async-scheduling: true - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - numa-bind: true - offload-group-size: 3 - offload-num-in-group: 1 - offload-prefetch-step: 2 - pipeline-parallel-size: 1 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - tensor-parallel-size: 1 - tokenizer-mode: deepseek_v4 - trust-remote-code: true -benchmark: - concurrencies: "1" - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - isl: 8192 - osl: 1024 - req_rate: "inf" - type: "sa-bench" - use_chat_template: true -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.23.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.23.0" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c32-c128.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c32-c128.yaml deleted file mode 100644 index dc2fd3bdaa..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c32-c128.yaml +++ /dev/null @@ -1,127 +0,0 @@ -name: "svf-vllm-disagg-b200-low-latency-c32-c128" -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" -dynamo: - install: true - wheel: "1.2.0.dev20260426" -setup_script: vllm-container-deps.sh -slurm: - time_limit: "8:00:00" -health_check: - interval_seconds: 10 - max_attempts: 1440 -sbatch_directives: - segment: "1" -resources: - decode_nodes: 1 - decode_workers: 1 - gpu_type: b200 - gpus_per_decode: 8 - gpus_per_node: 8 - gpus_per_prefill: 8 - prefill_nodes: 1 - prefill_workers: 1 -infra: - etcd_nats_dedicated_node: false -frontend: - enable_multiple_frontends: false - type: dynamo -backend: - set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids - connector: null - decode_environment: - NCCL_CUMEM_ENABLE: '1' - TILELANG_CLEANUP_TEMP_FILES: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_SERVER_DEV_MODE: '1' - prefill_environment: - NCCL_CUMEM_ENABLE: '1' - TILELANG_CLEANUP_TEMP_FILES: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' - VLLM_SERVER_DEV_MODE: '1' - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' - type: vllm - vllm_config: - decode: - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - enable-sleep-mode: true - gpu-memory-utilization: 0.9 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - max-cudagraph-capture-size: 128 - max-model-len: 9280 - max-num-batched-tokens: 128 - max-num-seqs: 128 - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - pipeline-parallel-size: 1 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - stream-interval: 50 - tensor-parallel-size: 8 - tokenizer-mode: deepseek_v4 - trust-remote-code: true - prefill: - block-size: 256 - data-parallel-rpc-port: 13345 - data-parallel-size: 8 - enable-expert-parallel: true - enable-sleep-mode: true - enforce-eager: true - gpu-memory-utilization: 0.8 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - max-model-len: 9280 - max-num-batched-tokens: 32768 - max-num-seqs: 16 - no-async-scheduling: true - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - numa-bind: true - offload-group-size: 3 - offload-num-in-group: 1 - offload-prefetch-step: 2 - pipeline-parallel-size: 1 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - tensor-parallel-size: 1 - tokenizer-mode: deepseek_v4 - trust-remote-code: true -benchmark: - concurrencies: "32x128" - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - isl: 8192 - osl: 1024 - req_rate: "inf" - type: "sa-bench" - use_chat_template: true -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.23.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.23.0" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c64.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c64.yaml deleted file mode 100644 index a1c93b179d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c64.yaml +++ /dev/null @@ -1,127 +0,0 @@ -name: "svf-vllm-disagg-b200-low-latency-c64" -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" -dynamo: - install: true - wheel: "1.2.0.dev20260426" -setup_script: vllm-container-deps.sh -slurm: - time_limit: "8:00:00" -health_check: - interval_seconds: 10 - max_attempts: 1440 -sbatch_directives: - segment: "1" -resources: - decode_nodes: 1 - decode_workers: 1 - gpu_type: b200 - gpus_per_decode: 8 - gpus_per_node: 8 - gpus_per_prefill: 8 - prefill_nodes: 1 - prefill_workers: 1 -infra: - etcd_nats_dedicated_node: false -frontend: - enable_multiple_frontends: false - type: dynamo -backend: - set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids - connector: null - decode_environment: - NCCL_CUMEM_ENABLE: '1' - TILELANG_CLEANUP_TEMP_FILES: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_SERVER_DEV_MODE: '1' - prefill_environment: - NCCL_CUMEM_ENABLE: '1' - TILELANG_CLEANUP_TEMP_FILES: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' - VLLM_SERVER_DEV_MODE: '1' - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' - type: vllm - vllm_config: - decode: - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - enable-sleep-mode: true - gpu-memory-utilization: 0.9 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - max-cudagraph-capture-size: 256 - max-model-len: 9280 - max-num-batched-tokens: 256 - max-num-seqs: 256 - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - pipeline-parallel-size: 1 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - stream-interval: 50 - tensor-parallel-size: 8 - tokenizer-mode: deepseek_v4 - trust-remote-code: true - prefill: - block-size: 256 - data-parallel-rpc-port: 13345 - data-parallel-size: 8 - enable-expert-parallel: true - enable-sleep-mode: true - enforce-eager: true - gpu-memory-utilization: 0.8 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - max-model-len: 9280 - max-num-batched-tokens: 32768 - max-num-seqs: 16 - no-async-scheduling: true - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - numa-bind: true - offload-group-size: 3 - offload-num-in-group: 1 - offload-prefetch-step: 2 - pipeline-parallel-size: 1 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - tensor-parallel-size: 1 - tokenizer-mode: deepseek_v4 - trust-remote-code: true -benchmark: - concurrencies: "64" - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - isl: 8192 - osl: 1024 - req_rate: "inf" - type: "sa-bench" - use_chat_template: true -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.23.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.23.0" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c256.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c256.yaml deleted file mode 100644 index fe4b49077b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c256.yaml +++ /dev/null @@ -1,131 +0,0 @@ -name: "svf-vllm-disagg-b200-low-middle-c256" -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" -dynamo: - install: true - wheel: "1.2.0.dev20260426" -setup_script: vllm-container-deps.sh -slurm: - time_limit: "8:00:00" -health_check: - interval_seconds: 10 - max_attempts: 1440 -sbatch_directives: - segment: "1" -resources: - decode_nodes: 1 - decode_workers: 1 - gpu_type: b200 - gpus_per_decode: 8 - gpus_per_node: 8 - gpus_per_prefill: 8 - prefill_nodes: 1 - prefill_workers: 1 -infra: - etcd_nats_dedicated_node: false -frontend: - enable_multiple_frontends: false - type: dynamo -backend: - set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids - connector: null - decode_environment: - NCCL_CUMEM_ENABLE: '1' - TILELANG_CLEANUP_TEMP_FILES: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' - VLLM_SERVER_DEV_MODE: '1' - prefill_environment: - NCCL_CUMEM_ENABLE: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TILELANG_CLEANUP_TEMP_FILES: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' - VLLM_SERVER_DEV_MODE: '1' - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' - type: vllm - vllm_config: - decode: - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - data-parallel-rpc-port: 13345 - data-parallel-size: 8 - enable-ep-weight-filter: true - enable-expert-parallel: true - enable-sleep-mode: true - gpu-memory-utilization: 0.9 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - max-cudagraph-capture-size: 256 - max-model-len: 9280 - max-num-batched-tokens: 256 - max-num-seqs: 256 - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - pipeline-parallel-size: 1 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - stream-interval: 50 - tensor-parallel-size: 1 - tokenizer-mode: deepseek_v4 - trust-remote-code: true - prefill: - block-size: 256 - data-parallel-rpc-port: 13345 - data-parallel-size: 8 - enable-ep-weight-filter: true - enable-expert-parallel: true - enable-sleep-mode: true - enforce-eager: true - gpu-memory-utilization: 0.95 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - max-model-len: 9280 - max-num-batched-tokens: 32768 - max-num-seqs: 16 - no-async-scheduling: true - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - numa-bind: true - pipeline-parallel-size: 1 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - tensor-parallel-size: 1 - tokenizer-mode: deepseek_v4 - trust-remote-code: true -benchmark: - concurrencies: "256" - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - isl: 8192 - osl: 1024 - req_rate: "inf" - type: "sa-bench" - use_chat_template: true -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.23.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.23.0" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c512.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c512.yaml deleted file mode 100644 index 8fe69b472f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c512.yaml +++ /dev/null @@ -1,132 +0,0 @@ -name: "svf-vllm-disagg-b200-low-middle-c512" -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" -dynamo: - install: true - wheel: "1.2.0.dev20260426" -setup_script: vllm-container-deps.sh -slurm: - time_limit: "8:00:00" -health_check: - max_attempts: 1440 - interval_seconds: 10 -sbatch_directives: - segment: "1" -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 -infra: - etcd_nats_dedicated_node: false -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: '1' - NCCL_CUMEM_ENABLE: '1' - VLLM_SERVER_DEV_MODE: '1' - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: '1' - NCCL_CUMEM_ENABLE: '1' - VLLM_SERVER_DEV_MODE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - enforce-eager: true - max-model-len: 9280 - max-num-seqs: 16 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "512" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.23.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.23.0" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-high-tpt-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-high-tpt-megamoe.yaml deleted file mode 100644 index b16d38d33d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-high-tpt-megamoe.yaml +++ /dev/null @@ -1,144 +0,0 @@ -name: "svf-vllm-disagg-b300-high-tpt-megamoe" - -# B300 adaptation of the DSV4 GB200/B200 vLLM disagg recipe. Each worker uses -# one full 8-GPU B300 node, plus a dedicated NATS/etcd infra node. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 2 - decode_nodes: 1 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - attention-config: '{"use_fp4_indexer_cache": true}' - enforce-eager: true - max-model-len: 9280 - max-num-seqs: 16 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - reasoning-parser: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - attention-config: '{"use_fp4_indexer_cache": true}' - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - gpu-memory-utilization: 0.85 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - reasoning-parser: deepseek_v4 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.23.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.23.0" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-low-latency.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-low-latency.yaml deleted file mode 100644 index 8f2a01f5c5..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-low-latency.yaml +++ /dev/null @@ -1,138 +0,0 @@ -name: "svf-vllm-disagg-b300-low-latency" - -# B300 adaptation of the DSV4 GB200/B200 vLLM disagg recipe. Each worker uses -# one full 8-GPU B300 node, plus a dedicated NATS/etcd infra node. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - attention-config: '{"use_fp4_indexer_cache": true}' - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 16 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - gpu-memory-utilization: 0.8 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - offload-group-size: 3 - offload-num-in-group: 1 - offload-prefetch-step: 2 - tokenizer-mode: deepseek_v4 - reasoning-parser: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - max-model-len: 16384 - max-num-seqs: 256 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - reasoning-parser: deepseek_v4 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1x32x64x128" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - container: - image: "vllm/vllm-openai:v0.23.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.23.0" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-mid-curve-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-mid-curve-megamoe.yaml deleted file mode 100644 index 557f6b1e95..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-mid-curve-megamoe.yaml +++ /dev/null @@ -1,144 +0,0 @@ -name: "svf-vllm-disagg-b300-mid-curve-megamoe" - -# B300 adaptation of the DSV4 GB200/B200 vLLM disagg recipe. Each worker uses -# one full 8-GPU B300 node, plus a dedicated NATS/etcd infra node. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - set_cuda_visible_devices: true # keep the aflowers/vllm-gb200-v0.20.0 provenance: bind GPUs via CVD, not --device-ids - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - attention-config: '{"use_fp4_indexer_cache": true}' - enforce-eager: true - max-model-len: 9280 - max-num-seqs: 16 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - reasoning-parser: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - attention-config: '{"use_fp4_indexer_cache": true}' - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - gpu-memory-utilization: 0.85 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - reasoning-parser: deepseek_v4 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256x1024" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.23.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.23.0" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe-mtp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe-mtp2.yaml deleted file mode 100644 index 24b134c1b9..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe-mtp2.yaml +++ /dev/null @@ -1,145 +0,0 @@ -name: "svf-vllm-disagg-gb200-high-tpt-megamoe-mtp2" - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.1-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - enforce-eager: true - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - attention-config: '{"use_fp4_indexer_cache":true}' - max-model-len: 9280 - max-num-seqs: 16 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.94 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - attention-config: '{"use_fp4_indexer_cache":true}' - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 1024 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1024" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.20.1-ubuntu2404" - frameworks: - dynamo: "1.2.0.dev20260426" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml deleted file mode 100644 index 2cc4cfc29a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml +++ /dev/null @@ -1,157 +0,0 @@ -name: "svf-vllm-disagg-gb200-high-tpt-megamoe" - -# Mirrored from NVIDIA/srt-slurm aflowers/vllm-gb200-v0.20.0 branch: -# recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-high-tpt-megamoe.yaml -# -# Topology: 2 prefill (DEP=8 each) + 1 decode (DEP=8). 7 nodes total with a -# dedicated NATS/etcd infra node. MegaMOE high-throughput point at concurrency -# 4096 with no CPU/NVMe offload. -# -# Local deltas vs upstream: -# * model.path uses the deepseek-v4-pro-mxfp4 alias from -# runners/launch_gb200-nv.sh. -# * model.container set to vllm/vllm-openai:v0.25.1 to -# match nvidia-master.yaml image (which the launch script registers as -# the alias key in srtslurm.yaml). Upstream variants ship either the -# non-dynamo floating tag or a sha256 pin. -# * slurm.time_limit + health_check set to 8h / 1440 attempts to -# absorb cold-cache /mnt/numa1 model loads. -model: - path: "deepseek-v4-pro-mxfp4" - container: "vllm/vllm-openai:v0.25.1" - precision: "fp4" - -dynamo: - install: true - wheel: "1.3.0.dev1" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: true - env: - DYN_REQUEST_PLANE_CODEC: "msgpack" -backend: - type: vllm - connector: null - prefill_environment: - DYN_REQUEST_PLANE_CODEC: "msgpack" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - decode_environment: - DYN_REQUEST_PLANE_CODEC: "msgpack" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - enforce-eager: true - max-model-len: 9280 - max-num-seqs: 16 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "b5968e9190ef611bbf34a7229255be88a0e937c1" - container: - image: "vllm/vllm-openai:v0.25.1" - frameworks: - dynamo: "1.3.0.dev1" - vllm: "0.25.1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency-mtp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency-mtp2.yaml deleted file mode 100644 index 7e0d09a0ef..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency-mtp2.yaml +++ /dev/null @@ -1,131 +0,0 @@ -name: svf-vllm-disagg-gb200-low-latency-mtp2 - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.1-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 8 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 8 - gpus_per_decode: 8 -infra: - etcd_nats_dedicated_node: true -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: '3600' - TILELANG_CLEANUP_TEMP_FILES: '1' - VLLM_USE_NCCL_SYMM_MEM: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - VLLM_SERVER_DEV_MODE: '1' - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_TLS: cuda_copy,cuda_ipc,tcp - UCX_CUDA_IPC_ENABLE_MNNVL: y - NCCL_P2P_LEVEL: NVL - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: '3600' - TILELANG_CLEANUP_TEMP_FILES: '1' - VLLM_USE_NCCL_SYMM_MEM: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - VLLM_SERVER_DEV_MODE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_TLS: cuda_copy,cuda_ipc,tcp - UCX_CUDA_IPC_ENABLE_MNNVL: y - NCCL_P2P_LEVEL: NVL - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-hybrid-lb: true - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enforce-eager: true - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - attention-config: '{"use_fp4_indexer_cache":true}' - max-model-len: 9280 - max-num-seqs: 8 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - attention-config: '{"use_fp4_indexer_cache":true}' - max-model-len: 9280 - max-num-seqs: 256 - max-cudagraph-capture-size: 256 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: 16x32x64 - req_rate: inf - use_chat_template: true - custom_tokenizer: sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.20.1-ubuntu2404" - frameworks: - dynamo: "1.2.0.dev20260426" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml deleted file mode 100644 index d4a35a6b7c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml +++ /dev/null @@ -1,154 +0,0 @@ -name: "svf-vllm-disagg-gb200-low-latency" - -# Mirrored from NVIDIA/srt-slurm aflowers/vllm-gb200-v0.20.0 branch: -# recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-low-latency.yaml -# -# Topology: 1 prefill (DEP=8) + 1 decode (TP=8). 5 nodes total with a -# dedicated NATS/etcd infra node. Single-concurrency point for low latency. -# -# Local deltas vs upstream: -# * model.path uses the deepseek-v4-pro-mxfp4 alias from -# runners/launch_gb200-nv.sh. -# * model.container set to vllm/vllm-openai:v0.25.1 to -# match nvidia-master.yaml image (which the launch script registers as -# the alias key in srtslurm.yaml). Upstream variants ship either the -# non-dynamo floating tag or a sha256 pin. -# * slurm.time_limit + health_check set to 8h / 1440 attempts to -# absorb cold-cache /mnt/numa1 model loads. -model: - path: "deepseek-v4-pro-mxfp4" - container: "vllm/vllm-openai:v0.25.1" - precision: "fp4" - -dynamo: - install: true - wheel: "1.3.0.dev1" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: true - env: - DYN_REQUEST_PLANE_CODEC: "msgpack" -backend: - type: vllm - connector: null - prefill_environment: - DYN_REQUEST_PLANE_CODEC: "msgpack" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - decode_environment: - DYN_REQUEST_PLANE_CODEC: "msgpack" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 16 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.8 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - offload-group-size: 3 - offload-num-in-group: 1 - offload-prefetch-step: 2 - # offload-params: "w13_weight w2_weight w13_weight_scale w2_weight_scale wq_b wo_a wo_b shared_experts" - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 -# data-parallel-size: 8 -# data-parallel-rpc-port: 13345 -# enable-expert-parallel: true - max-model-len: 16384 - max-num-seqs: 256 - max-cudagraph-capture-size: 256 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "b5968e9190ef611bbf34a7229255be88a0e937c1" - container: - image: "vllm/vllm-openai:v0.25.1" - frameworks: - dynamo: "1.3.0.dev1" - vllm: "0.25.1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml deleted file mode 100644 index 9295c201da..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "svf-vllm-disagg-gb200-low-middle-curve" - -# Mirrored from NVIDIA/srt-slurm aflowers/vllm-gb200-v0.20.0 branch: -# recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-low-middle-curve.yaml -# -# Topology: 1 prefill (DEP=8) + 4 decode (TP=8). 11 nodes total with a -# dedicated NATS/etcd infra node. Low-middle curve points at concurrencies -# 256 and 512. -# -# Local deltas vs upstream: -# * model.path uses the deepseek-v4-pro-mxfp4 alias from -# runners/launch_gb200-nv.sh. -# * model.container set to vllm/vllm-openai:v0.25.1 to -# match nvidia-master.yaml image (which the launch script registers as -# the alias key in srtslurm.yaml). Upstream variants ship either the -# non-dynamo floating tag or a sha256 pin. -# * slurm.time_limit + health_check set to 8h / 1440 attempts to -# absorb cold-cache /mnt/numa1 model loads. -model: - path: "deepseek-v4-pro-mxfp4" - container: "vllm/vllm-openai:v0.25.1" - precision: "fp4" - -dynamo: - install: true - wheel: "1.3.0.dev1" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 8 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: true - env: - DYN_REQUEST_PLANE_CODEC: "msgpack" -backend: - type: vllm - connector: null - prefill_environment: - DYN_REQUEST_PLANE_CODEC: "msgpack" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - decode_environment: - DYN_REQUEST_PLANE_CODEC: "msgpack" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 16 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.8 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - offload-group-size: 3 - offload-num-in-group: 1 - offload-prefetch-step: 2 - # offload-params: "w13_weight w2_weight w13_weight_scale w2_weight_scale wq_b wo_a wo_b shared_experts" - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 -# data-parallel-size: 8 -# data-parallel-rpc-port: 13345 -# enable-expert-parallel: true - max-model-len: 16384 - max-num-seqs: 256 - max-cudagraph-capture-size: 256 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.85 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256x512" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "b5968e9190ef611bbf34a7229255be88a0e937c1" - container: - image: "vllm/vllm-openai:v0.25.1" - frameworks: - dynamo: "1.3.0.dev1" - vllm: "0.25.1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml deleted file mode 100644 index 29511c43cc..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml +++ /dev/null @@ -1,157 +0,0 @@ -name: "svf-vllm-disagg-gb200-max-tpt-megamoe" - -# Mirrored from NVIDIA/srt-slurm aflowers/vllm-gb200-v0.20.0 branch: -# recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-max-tpt-megamoe.yaml -# -# Topology: 3 prefill (DEP=8 each) + 1 decode (DEP=8). 9 nodes total with a -# dedicated NATS/etcd infra node. MegaMOE max-throughput point at concurrency -# 4096 with no CPU/NVMe offload. -# -# Local deltas vs upstream: -# * model.path uses the deepseek-v4-pro-mxfp4 alias from -# runners/launch_gb200-nv.sh. -# * model.container set to vllm/vllm-openai:v0.25.1 to -# match nvidia-master.yaml image (which the launch script registers as -# the alias key in srtslurm.yaml). Upstream variants ship either the -# non-dynamo floating tag or a sha256 pin. -# * slurm.time_limit + health_check set to 8h / 1440 attempts to -# absorb cold-cache /mnt/numa1 model loads. -model: - path: "deepseek-v4-pro-mxfp4" - container: "vllm/vllm-openai:v0.25.1" - precision: "fp4" - -dynamo: - install: true - wheel: "1.3.0.dev1" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 6 - decode_nodes: 2 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: true - env: - DYN_REQUEST_PLANE_CODEC: "msgpack" -backend: - type: vllm - connector: null - prefill_environment: - DYN_REQUEST_PLANE_CODEC: "msgpack" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - decode_environment: - DYN_REQUEST_PLANE_CODEC: "msgpack" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - enforce-eager: true - max-model-len: 9280 - max-num-seqs: 16 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "b5968e9190ef611bbf34a7229255be88a0e937c1" - container: - image: "vllm/vllm-openai:v0.25.1" - frameworks: - dynamo: "1.3.0.dev1" - vllm: "0.25.1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe-mtp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe-mtp2.yaml deleted file mode 100644 index 813584eb90..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe-mtp2.yaml +++ /dev/null @@ -1,145 +0,0 @@ -name: "svf-vllm-disagg-gb200-mid-curve-megamoe-mtp2" - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.21.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-hybrid-lb: true - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - enforce-eager: true - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - attention-config: '{"use_fp4_indexer_cache":true}' - max-model-len: 9280 - max-num-seqs: 8 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-hybrid-lb: true - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - attention-config: '{"use_fp4_indexer_cache":true}' - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "128x256x512x1024" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.21.0-ubuntu2404" - frameworks: - dynamo: "1.2.0.dev20260426" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml deleted file mode 100644 index 59bf39b448..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml +++ /dev/null @@ -1,157 +0,0 @@ -name: "svf-vllm-disagg-gb200-mid-curve-megamoe" - -# Mirrored from NVIDIA/srt-slurm aflowers/vllm-gb200-v0.20.0 branch: -# recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-mid-curve-megamoe.yaml -# -# Topology: 1 prefill (DEP=8) + 1 decode (DEP=8). 5 nodes total with a -# dedicated NATS/etcd infra node. MegaMOE mid-curve points at concurrency -# 256/512/1024 with no CPU/NVMe offload. -# -# Local deltas vs upstream: -# * model.path uses the deepseek-v4-pro-mxfp4 alias from -# runners/launch_gb200-nv.sh. -# * model.container set to vllm/vllm-openai:v0.25.1 to -# match nvidia-master.yaml image (which the launch script registers as -# the alias key in srtslurm.yaml). Upstream variants ship either the -# non-dynamo floating tag or a sha256 pin. -# * slurm.time_limit + health_check set to 8h / 1440 attempts to -# absorb cold-cache /mnt/numa1 model loads. -model: - path: "deepseek-v4-pro-mxfp4" - container: "vllm/vllm-openai:v0.25.1" - precision: "fp4" - -dynamo: - install: true - wheel: "1.3.0.dev1" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: true - env: - DYN_REQUEST_PLANE_CODEC: "msgpack" -backend: - type: vllm - connector: null - prefill_environment: - DYN_REQUEST_PLANE_CODEC: "msgpack" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - decode_environment: - DYN_REQUEST_PLANE_CODEC: "msgpack" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - enforce-eager: true - max-model-len: 9280 - max-num-seqs: 16 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256x512x1024" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "b5968e9190ef611bbf34a7229255be88a0e937c1" - container: - image: "vllm/vllm-openai:v0.25.1" - frameworks: - dynamo: "1.3.0.dev1" - vllm: "0.25.1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p6d-dep4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p6d-dep4-tp4.yaml deleted file mode 100644 index 7a299d81a5..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p6d-dep4-tp4.yaml +++ /dev/null @@ -1,116 +0,0 @@ -name: "svf-vllm-disagg-gb300-1p6d-dep4-tp4" - -# Topology: 1 prefill (DEP=4) + 6 decode (TP=4). 7 GB300 nodes (1P + 6D = 28 -# GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra node. Low-mid curve -# point at concurrency 192. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 6 - prefill_workers: 1 - decode_workers: 6 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - attention-config: '{"use_fp4_indexer_cache": true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 256 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.9 - enable-ep-weight-filter: true - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - max-model-len: 16384 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-ep-weight-filter: true - all2all-backend: "flashinfer_nvlink_one_sided" - no-enable-flashinfer-autotune: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "192" - req_rate: "inf" - tokenizer_mode: "deepseek_v4" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p9d-tep4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p9d-tep4-tp4.yaml deleted file mode 100644 index d1e895bbbf..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p9d-tep4-tp4.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "svf-vllm-disagg-gb300-1p9d-tep4-tp4" - -# Topology: 1 prefill (TEP=4) + 9 decode (TP=4). 10 GB300 nodes (1P + 9D = 40 -# GPUs at 4 GPUs/node), NATS/etcd colocated on the prefill node. -# Wide-decode point at concurrency 18 — each decode worker holds a -# single replica. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 9 - prefill_workers: 1 - decode_workers: 9 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: true - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 256 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.9 - enable-ep-weight-filter: true - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - max-model-len: 16384 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-ep-weight-filter: true - all2all-backend: "flashinfer_nvlink_one_sided" - no-enable-flashinfer-autotune: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "18" - req_rate: "inf" - tokenizer_mode: "deepseek_v4" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-4p1d-dep4-dep8-24-c4096.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-4p1d-dep4-dep8-24-c4096.yaml deleted file mode 100644 index 0a3643cf27..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-4p1d-dep4-dep8-24-c4096.yaml +++ /dev/null @@ -1,125 +0,0 @@ -name: "svf-vllm-disagg-gb300-4p1d-dep4-dep8-24" - -# Topology: 4 prefill (DEP=4 each) + 1 decode (DEP=8). 6 GB300 nodes (4P + 2D -# = 24 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra node. -# Max-throughput point at concurrency 4096 with deep_gemm_amxf4_mega_moe on -# both workers. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" - precision: "fp4" - -dynamo: - wheel: "1.2.0.dev20260426" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 2 - prefill_workers: 4 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - safetensors-load-strategy: "prefetch" - block-size: 256 - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - no-async-scheduling: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_amxf4_mega_moe" - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 16384 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_amxf4_mega_moe" - no-enable-flashinfer-autotune: true - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" - tokenizer_mode: "deepseek_v4" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-5p1d-dep4-dep8-28-c4096.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-5p1d-dep4-dep8-28-c4096.yaml deleted file mode 100644 index 698ef94e2b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-5p1d-dep4-dep8-28-c4096.yaml +++ /dev/null @@ -1,125 +0,0 @@ -name: "svf-vllm-disagg-gb300-5p1d-dep4-dep8-28" - -# Topology: 5 prefill (DEP=4 each) + 1 decode (DEP=8). 7 GB300 nodes (5P + 2D -# = 28 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra node. -# Max-throughput point at concurrency 4096 with deep_gemm_amxf4_mega_moe on -# both workers. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" - precision: "fp4" - -dynamo: - wheel: "1.2.0.dev20260426" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 5 - decode_nodes: 2 - prefill_workers: 5 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - safetensors-load-strategy: "prefetch" - block-size: 256 - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - no-async-scheduling: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_amxf4_mega_moe" - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 16384 - max-num-seqs: 384 - max-cudagraph-capture-size: 512 - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_amxf4_mega_moe" - no-enable-flashinfer-autotune: true - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" - tokenizer_mode: "deepseek_v4" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-6p1d-dep4-dep8-32-c4096.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-6p1d-dep4-dep8-32-c4096.yaml deleted file mode 100644 index 85a1d4c6b6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-6p1d-dep4-dep8-32-c4096.yaml +++ /dev/null @@ -1,125 +0,0 @@ -name: "svf-vllm-disagg-gb300-6p1d-dep4-dep8-32" - -# Topology: 6 prefill (DEP=4 each) + 1 decode (DEP=8). 8 GB300 nodes (6P + 2D -# = 32 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra node. -# Max-throughput point at concurrency 4096 with deep_gemm_amxf4_mega_moe on -# both workers. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" - precision: "fp4" - -dynamo: - wheel: "1.2.0.dev20260426" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 6 - decode_nodes: 2 - prefill_workers: 6 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - safetensors-load-strategy: "prefetch" - block-size: 256 - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - no-async-scheduling: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_amxf4_mega_moe" - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 16384 - max-num-seqs: 384 - max-cudagraph-capture-size: 512 - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_amxf4_mega_moe" - no-enable-flashinfer-autotune: true - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" - tokenizer_mode: "deepseek_v4" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-7p2d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-7p2d-dep4-dep16.yaml deleted file mode 100644 index 53534f9697..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-7p2d-dep4-dep16.yaml +++ /dev/null @@ -1,122 +0,0 @@ -name: "svf-vllm-disagg-gb300-7p2d-dep4-dep16" - -# Topology: 7 prefill (DEP=4) + 2 decode (DEP=16). 15 GB300 nodes (7P + 8D -# = 60 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra node. -# Wide-EP decode max-throughput point at concurrency 3072. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:dsv4-megamoe-mxfp4-arm64-cu130-4ba0a72" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 7 - decode_nodes: 8 - prefill_workers: 7 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 16 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - attention-config: '{"use_fp4_indexer_cache": true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 256 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.9 - enable-ep-weight-filter: true - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - attention-config: '{"use_fp4_indexer_cache": true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - max-model-len: 16384 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-ep-weight-filter: true - all2all-backend: "flashinfer_nvlink_one_sided" - no-enable-flashinfer-autotune: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "3072" - req_rate: "inf" - tokenizer_mode: "deepseek_v4" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml deleted file mode 100644 index 7c2ae05fc3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml +++ /dev/null @@ -1,102 +0,0 @@ -name: dsv4-gb200-vllm-agentic-mtp-agg-dep8 - -model: - path: deepseek-v4-pro - container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - precision: fp4 - -identity: - model: { repo: deepseek-ai/DeepSeek-V4-Pro } - container: { image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 } - frameworks: { dynamo: "1.3.1" } - -dynamo: - version: "1.3.1" - install: true - -setup_script: vllm-container-deps.sh -environment: { ETCD_LEASE_TTL: "7200" } -slurm: { time_limit: "8:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } - -resources: - gpu_type: gb200 - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: kv - router-reset-states: true - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 256 - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - aggregated_environment: - HF_HUB_CACHE: /hf_hub_cache - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - vllm_config: - aggregated: - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 8192 - trust-remote-code: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - gpu-memory-utilization: 0.90 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - kv-cache-metrics: true - tokenizer-mode: deepseek_v4 - -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-c4-mtp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-c4-mtp2-agentic.yaml deleted file mode 100644 index e98caaa3c3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-c4-mtp2-agentic.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: "svf-vllm-agg-gb200-tp8-c4-mtp2-agentic" - -# GB200 AgentX aggregate topology: one TP8 worker spans two four-GPU -# nodes and serves both prefill and decode at concurrency 4. Keep at least -# 16 sequence slots and otherwise size the scheduler at 4x concurrency. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.2.1" - -dynamo: - wheel: "1.2.1" - install: true - -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - # The frontend shares Grace CPU capacity with the long TP8 cold start. - ETCD_LEASE_TTL: "7200" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-agg-mtp2-{job_id}" - vllm_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 16 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.94 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep aggregate workers in the multinode result schema so ingestion uses - # the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-c8-mtp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-c8-mtp2-agentic.yaml deleted file mode 100644 index 02aed89abb..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-c8-mtp2-agentic.yaml +++ /dev/null @@ -1,147 +0,0 @@ -name: "svf-vllm-agg-gb200-tp8-c8-mtp2-agentic" - -# GB200 AgentX aggregate topology: one TP8 worker spans two four-GPU -# nodes and serves both prefill and decode at concurrency 8. Size max-num-seqs at -# 4x concurrency and expand the MTP CUDA-graph envelope to match. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.2.1" - -dynamo: - wheel: "1.2.1" - install: true - -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - ETCD_LEASE_TTL: "7200" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-agg-tp8-c8-mtp2-{job_id}" - vllm_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.94 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml deleted file mode 100644 index f35d7c6bc8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml +++ /dev/null @@ -1,112 +0,0 @@ -name: dsv4-gb200-vllm-agentic-mtp-agg-tp8 - -model: - path: deepseek-v4-pro - container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - precision: fp4 - -identity: - model: - repo: deepseek-ai/DeepSeek-V4-Pro - container: - image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - frameworks: - dynamo: "1.3.1" - -dynamo: - version: "1.3.1" - install: true - -setup_script: vllm-container-deps.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: kv - router-reset-states: true - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 256 - -backend: - type: vllm - connector: null - aggregated_environment: - HF_HUB_CACHE: /hf_hub_cache - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - vllm_config: - aggregated: - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 128 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - gpu-memory-utilization: 0.90 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - kv-cache-metrics: true - tokenizer-mode: deepseek_v4 - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml deleted file mode 100644 index 9559ffb75b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ /dev/null @@ -1,147 +0,0 @@ -name: "svf-vllm-agg-gb300-tp4-mtp-agentic" - -# GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node -# and serves both prefill and decode at concurrency 8. Size max-num-seqs at -# 4x concurrency and expand the MTP CUDA-graph envelope to match. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.2.1" - -dynamo: - wheel: "1.2.1" - install: true - -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - ETCD_LEASE_TTL: "7200" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp4-mtp-{job_id}" - vllm_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.94 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml deleted file mode 100644 index 4eebce98fa..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: "svf-vllm-agg-gb300-tp8-mtp-agentic" - -# GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU -# nodes and serves both prefill and decode at concurrency 4. Keep at least -# 16 sequence slots and otherwise size the scheduler at 4x concurrency. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.2.1" - -dynamo: - wheel: "1.2.1" - install: true - -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - # The frontend shares Grace CPU capacity with the long TP8 cold start. - ETCD_LEASE_TTL: "7200" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-mtp-{job_id}" - vllm_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 16 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.94 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep aggregate workers in the multinode result schema so ingestion uses - # the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-c128-mtp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-c128-mtp2-agentic.yaml deleted file mode 100644 index 675f469ea7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-c128-mtp2-agentic.yaml +++ /dev/null @@ -1,208 +0,0 @@ -name: "svf-vllm-disagg-gb200-1p1d-dep8-dep8-c128-mtp2-agentic" - -# GB200 AgentX MTP3 topology: one DEP8 prefill worker feeds one -# DEP8 decode worker at concurrency 128. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - wheel: "1.3.0.dev20260720" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "140GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 128 - max-num-batched-tokens: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 512 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - prefill_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c128-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - decode_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c128-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-c256-mtp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-c256-mtp2-agentic.yaml deleted file mode 100644 index 9e01da1746..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-c256-mtp2-agentic.yaml +++ /dev/null @@ -1,208 +0,0 @@ -name: "svf-vllm-disagg-gb200-1p1d-dep8-dep8-c256-mtp2-agentic" - -# GB200 AgentX MTP3 topology: one DEP8 prefill worker feeds one -# DEP8 decode worker at concurrency 256. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - wheel: "1.3.0.dev20260720" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "140GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 128 - max-num-batched-tokens: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 512 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - prefill_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - decode_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-1p1d-dep8-dep8-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml deleted file mode 100644 index 221215b33f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml +++ /dev/null @@ -1,122 +0,0 @@ -name: dsv4-gb200-vllm-agentic-mtp-disagg-1p1d-dep8-dep8 - -model: - path: deepseek-v4-pro - container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - precision: fp4 - -identity: - model: { repo: deepseek-ai/DeepSeek-V4-Pro } - container: { image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 } - frameworks: { dynamo: "1.3.1" } - -dynamo: - version: "1.3.1" - install: true - -setup_script: vllm-container-deps.sh -environment: { ETCD_LEASE_TTL: "7200" } -slurm: { time_limit: "8:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - # Long AgentX prefills can exceed Dynamo's request-plane default while - # the healthy DEP8 worker is still computing the first response. - DYN_TCP_REQUEST_TIMEOUT: "60" - args: - router-mode: kv - router-reset-states: true - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 256 - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - prefill_environment: &worker_environment - HF_HUB_CACHE: /hf_hub_cache - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - decode_environment: *worker_environment - vllm_config: - prefill: &dep8_config - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - enforce-eager: true - block-size: 256 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - kv-cache-metrics: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - decode: - <<: *dep8_config - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' - enforce-eager: false - gpu-memory-utilization: 0.90 - max-num-seqs: 1024 - max-num-batched-tokens: 1024 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 1024 - stream-interval: 10 - -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-c576-mtp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-c576-mtp2-agentic.yaml deleted file mode 100644 index 7ab7977171..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-c576-mtp2-agentic.yaml +++ /dev/null @@ -1,210 +0,0 @@ -name: "svf-vllm-disagg-gb200-2p1d-dep8-dep12-c576-mtp2-agentic" - -# GB200 AgentX MTP3 topology: two DEP8 prefill workers feed one -# DEP12 decode worker at concurrency 576. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - wheel: "1.3.0.dev20260720" - install: true - -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 4 - decode_nodes: 3 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 12 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "140GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 12 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - prefill_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-prefill-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - decode_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-dep12-decode-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep16-c512-mtp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep16-c512-mtp2-agentic.yaml deleted file mode 100644 index 1690273d14..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep16-c512-mtp2-agentic.yaml +++ /dev/null @@ -1,210 +0,0 @@ -name: "svf-vllm-disagg-gb200-2p1d-dep8-dep16-c512-mtp2-agentic" - -# GB200 AgentX MTP3 topology: two DEP8 prefill workers feed one -# DEP16 decode worker at concurrency 512. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - wheel: "1.3.0.dev20260720" - install: true - -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 4 - decode_nodes: 4 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "140GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - prefill_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-prefill-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - decode_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb200-2p1d-dep16-decode-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml deleted file mode 100644 index 8b3ddfe146..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml +++ /dev/null @@ -1,117 +0,0 @@ -name: dsv4-gb200-vllm-agentic-mtp-disagg-2p1d-dep8-dep8 - -model: - path: deepseek-v4-pro - container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 - precision: fp4 - -identity: - model: { repo: deepseek-ai/DeepSeek-V4-Pro } - container: { image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 } - frameworks: { dynamo: "1.3.1" } - -dynamo: { version: "1.3.1", install: true } -setup_script: vllm-container-deps.sh -environment: { ETCD_LEASE_TTL: "7200" } -slurm: { time_limit: "8:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - # Long AgentX prefills can exceed Dynamo's request-plane default while - # the healthy DEP8 workers are still computing their first responses. - DYN_TCP_REQUEST_TIMEOUT: "60" - args: - router-mode: kv - router-reset-states: true - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 256 - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - prefill_environment: &worker_environment - HF_HUB_CACHE: /hf_hub_cache - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - decode_environment: *worker_environment - vllm_config: - prefill: &dep8_config - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - enforce-eager: true - block-size: 256 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - kv-cache-metrics: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - decode: - <<: *dep8_config - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' - enforce-eager: false - gpu-memory-utilization: 0.90 - max-num-seqs: 1024 - max-num-batched-tokens: 1024 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 1024 - stream-interval: 10 - -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache - HF_HUB_CACHE: /hf_hub_cache - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c128-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c128-mtp-agentic.yaml deleted file mode 100644 index 4f7b03ca6a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c128-mtp-agentic.yaml +++ /dev/null @@ -1,215 +0,0 @@ -# Derived from PR #2302's DEP8/DEP32 c256 recipe. -# Halve the prefill/decode topology while preserving its conservative -# 4/16/16 decode limits and isolated JIT cache paths. -name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c128-mtp-agentic" - -# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one -# DEP16 decode worker at concurrency 128. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.4.0" - -dynamo: - wheel: "1.4.0" - install: true - -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 4 - max-num-batched-tokens: 16 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 16 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - prefill_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c128-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - decode_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c128-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml deleted file mode 100644 index cd782b4a98..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml +++ /dev/null @@ -1,215 +0,0 @@ -# Derived from PR #2302's DEP8/DEP32 c256 recipe. -# Halve the prefill/decode topology while preserving its conservative -# 8/32/32 decode limits and isolated JIT cache paths. -name: "svf-vllm-disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic" - -# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one -# DEP16 decode worker at concurrency 256. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.4.0" - -dynamo: - wheel: "1.4.0" - install: true - -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 8 - max-num-batched-tokens: 32 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 32 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - prefill_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - decode_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml deleted file mode 100644 index e5cc1c76ea..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml +++ /dev/null @@ -1,208 +0,0 @@ -name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic" - -# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one -# DEP8 decode worker at concurrency 256. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - wheel: "1.3.0.dev20260720" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 1024 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - prefill_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - decode_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml deleted file mode 100644 index f632eec5ee..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml +++ /dev/null @@ -1,215 +0,0 @@ -# Source: https://github.com/SemiAnalysisAI/InferenceX/blob/0c33d4615792705ed12bfc204e3a54cfa436cf02/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml -# Runtime and performance arguments follow the current DEP8/DEP16 GB300 P/D -# recipes; throughput-only synthetic MTP acceptance is injected at launch. -name: "svf-vllm-disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic" - -# GB300 AgentX MTP3 topology: one DEP8 prefill worker feeds one -# DEP16 decode worker at concurrency 512. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.4.0" - -dynamo: - wheel: "1.4.0" - install: true - -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 2 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 16384 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - prefill_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c512-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - decode_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep16-decode-c512-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tp8-c4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tp8-c4-mtp-agentic.yaml deleted file mode 100644 index 18e1e72fbd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tp8-c4-mtp-agentic.yaml +++ /dev/null @@ -1,224 +0,0 @@ -# Source: https://github.com/Inferact/srt-slurm-sa/blob/4a870cd5bc333bf8a312fc65ca1ea82cdab9b2df/recipes/vllm/deepseek-v4-pro/GB300/agentic/agentx-v1.0.1/1p3d-pdep4-dtp8-c3-kv-nixl-mtp-flashinfer-ar-lpt512-psi1.yaml -# Adapted from the source topology to a 1P4D concurrency-4 point for the -# InferenceX AgentX harness; eval-only runs continue to verify real MTP output. -name: "svf-vllm-disagg-gb300-1p4d-dep4-tp8-c4-mtp-agentic" - -# GB300 high-interactivity AgentX MTP3 topology: one DEP4 prefill worker -# feeds four TP8 decode workers at concurrency 4 through NIXL. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.4.0" - -dynamo: - install: true - wheel: "1.4.0" - request_plane: tcp - -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p4d-{job_id}" - # Multi-node TP8 decode spans two GB300 nodes; match the sibling recipes' - # NCCL/UCX fabric settings (MNNVL/NVLS, IB HCAs). - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - -slurm: - time_limit: "08:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 1 - decode_nodes: 8 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - MODEL: "deepseek-ai/DeepSeek-V4-Pro" - MODEL_PREFIX: "dsv4" - FRAMEWORK: "dynamo-vllm" - PRECISION: "fp4" - CONC: "4" - DURATION: "3600" - KV_OFFLOADING: "none" - ETCD_LEASE_TTL: "120" - DYN_ROUTER_QUEUE_THRESHOLD: "None" - DYN_TOKENIZER_CACHE: "1" - DYN_TOKENIZER_CACHE_BYTES: "8000000000" - DYN_TOKENIZER: "fastokens" - DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: "14400" - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - args: - router-mode: "kv" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - orchestrator_placement: first_decode - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - # vLLM KV routing needs prefill KV events so Dynamo can select the - # cache-owning PDEP4 rank. - kv_events_config: - prefill: true - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 512 - prefill-schedule-interval: 1 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - # MTP capture sizes are tokens: 64 seqs * (1 target + 3 drafts). - max-cudagraph-capture-size: 256 - decode: - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}}' - # TP8 spans two GB300 nodes; custom all-reduce is single-node only, so - # decode uses the FlashInfer allreduce path like the agg TP8 recipes. - disable-custom-all-reduce: true - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - max-model-len: 1048576 - max-num-seqs: 16 - max-num-batched-tokens: 64 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - # MTP capture sizes are tokens: 16 seqs * (1 target + 3 drafts). - max-cudagraph-capture-size: 64 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - prefill_environment: - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - -setup_script: vllm-container-deps.sh - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - # The AgentX client uses localhost:8000, so colocate it with the Dynamo - # frontend launched on the first decode node. - client_placement: first_decode - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - MODEL: "deepseek-ai/DeepSeek-V4-Pro" - MODEL_PREFIX: "dsv4" - SERVED_MODEL_NAME: "deepseek-ai/DeepSeek-V4-Pro" - FRAMEWORK: "dynamo-vllm" - PRECISION: "fp4" - CONC: "4" - DURATION: "3600" - RUNNER_TYPE: "gb300" - IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - SPEC_DECODING: "mtp" - DISAGG: "true" - OFFLOADING: "none" - KV_OFFLOADING: "none" - TP: "8" - PREFILL_TP: "1" - PREFILL_NUM_WORKERS: "1" - PREFILL_EP: "4" - DECODE_TP: "8" - DECODE_NUM_WORKERS: "4" - DECODE_EP: "1" - EP_SIZE: "1" - AIPERF_MAX_OSL: "none" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" - NUM_DATASET_ENTRIES: "393" - HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" - PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml deleted file mode 100644 index d5a0b44161..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml +++ /dev/null @@ -1,210 +0,0 @@ -name: "svf-vllm-disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic" - -# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one -# DEP12 decode worker at concurrency 1152. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - wheel: "1.3.0.dev20260720" - install: true - -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 4 - decode_nodes: 3 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 12 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 12 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - prefill_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - decode_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep12-decode-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml deleted file mode 100644 index f770241932..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml +++ /dev/null @@ -1,210 +0,0 @@ -name: "svf-vllm-disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic" - -# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one -# DEP16 decode worker at concurrency 1024. Decode consumes P/D KV through NIXL -# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - wheel: "1.3.0.dev20260720" - install: true - -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 4 - decode_nodes: 4 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TCP_CHANNEL_BUFFER: "128" - DYN_TCP_REQUEST_TIMEOUT: "60" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "180GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - long-prefill-token-threshold: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - prefill_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_MOONCAKE_STORE_SEND_THREADS: "8" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - decode_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep16-decode-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p1d-dep4-dep16.yaml deleted file mode 100644 index 0ee4f3e7bc..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p1d-dep4-dep16.yaml +++ /dev/null @@ -1,100 +0,0 @@ -name: "vllm-disagg-gb200-1p1d-dep4-dep16" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.2.1 - install: true - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 3072 - max-num-seqs: 1024 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.9 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 3072 - max-num-seqs: 256 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - all2all-backend: "flashinfer_nvlink_one_sided" - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 128 - max-cudagraph-capture-size: 384 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4096x6144" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p1d-dep4-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p1d-dep4-dep8.yaml deleted file mode 100644 index 40683d1b8d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p1d-dep4-dep8.yaml +++ /dev/null @@ -1,100 +0,0 @@ -name: "vllm-disagg-gb200-1p1d-dep4-dep8" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.2.1 - install: true - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 3072 - max-num-seqs: 1024 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.93 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 3072 - max-num-seqs: 1024 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - all2all-backend: "flashinfer_nvlink_one_sided" - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 128 - max-cudagraph-capture-size: 1024 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4096x12288" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p4d-dep4-tp8.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p4d-dep4-tp8.yaml deleted file mode 100644 index bda68b9207..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p4d-dep4-tp8.yaml +++ /dev/null @@ -1,96 +0,0 @@ -name: "kimi-vllm-disagg-gb200-1p4d-dep4-tp8" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.2.1 - install: true - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 8 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 3072 - max-num-seqs: 1024 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.93 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - enable-expert-parallel: false - max-model-len: 3072 - max-num-seqs: 1024 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 1024 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4x8x32x128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-1p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-1p1d-dep4-dep16.yaml deleted file mode 100644 index 7188a5a082..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-1p1d-dep4-dep16.yaml +++ /dev/null @@ -1,99 +0,0 @@ -name: "kimi-vllm-disagg-gb300-1p1d-dep4-dep16" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.3.0.dev20260601 - install: true - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 3072 - max-num-seqs: 1024 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.9 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 3072 - max-num-seqs: 192 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - all2all-backend: "flashinfer_nvlink_one_sided" - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 256 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "2048x4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-1p1d-dep4-dep24.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-1p1d-dep4-dep24.yaml deleted file mode 100644 index 79758b546f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-1p1d-dep4-dep24.yaml +++ /dev/null @@ -1,99 +0,0 @@ -name: "kimi-vllm-disagg-gb300-1p1d-dep4-dep24" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.3.0.dev20260601 - install: true - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 6 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 24 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 3072 - max-num-seqs: 1024 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.9 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 24 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 3072 - max-num-seqs: 86 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - all2all-backend: "flashinfer_nvlink_one_sided" - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 128 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1024" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-1p2d-dep4-dep4.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-1p2d-dep4-dep4.yaml deleted file mode 100644 index 15d4169abc..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-1p2d-dep4-dep4.yaml +++ /dev/null @@ -1,99 +0,0 @@ -name: "kimi-vllm-disagg-gb300-1p2d-dep4-dep4" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.3.0.dev20260601 - install: true - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 3072 - max-num-seqs: 1024 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.9 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 3072 - max-num-seqs: 768 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - all2all-backend: "flashinfer_nvlink_one_sided" - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 1024 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "6144" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-1p7d-tep4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-1p7d-tep4-tp4.yaml deleted file mode 100644 index a0ba50b871..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-1p7d-tep4-tp4.yaml +++ /dev/null @@ -1,94 +0,0 @@ -name: "kimi-vllm-disagg-gb300-1p7d-tep4-tp4" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.3.0.dev20260601 - install: true - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 7 - prefill_workers: 1 - decode_workers: 7 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: true - max-model-len: 3072 - max-num-seqs: 1024 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.9 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - max-model-len: 3072 - max-num-seqs: 1024 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - all2all-backend: "flashinfer_nvlink_one_sided" - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 1024 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "8x16x32x64x128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-2p3d-dep4-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-2p3d-dep4-dep8.yaml deleted file mode 100644 index cfa6037040..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-2p3d-dep4-dep8.yaml +++ /dev/null @@ -1,99 +0,0 @@ -name: "kimi-vllm-disagg-gb300-2p3d-dep4-dep8" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.3.0.dev20260601 - install: true - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 6 - prefill_workers: 2 - decode_workers: 3 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 3072 - max-num-seqs: 1024 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.9 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 3072 - max-num-seqs: 512 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - all2all-backend: "flashinfer_nvlink_one_sided" - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 1024 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-1p4d-dep4-tep4.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-1p4d-dep4-tep4.yaml deleted file mode 100644 index 429945c05e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-1p4d-dep4-tep4.yaml +++ /dev/null @@ -1,96 +0,0 @@ -name: "kimi-vllm-disagg-gb200-1p4d-dep4-tep4" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.2.1 - install: true - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 64 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.93 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 16 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 16 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-1p4d-dep4-tp8.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-1p4d-dep4-tp8.yaml deleted file mode 100644 index 55165aeb3b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-1p4d-dep4-tp8.yaml +++ /dev/null @@ -1,96 +0,0 @@ -name: "kimi-vllm-disagg-gb200-1p4d-dep4-tp8" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.2.1 - install: true - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 8 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 64 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.93 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - enable-expert-parallel: false - max-model-len: 10240 - max-num-seqs: 16 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 16 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4x8x16x32x256" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-3p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-3p1d-dep4-dep16.yaml deleted file mode 100644 index b4d1e8c82e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-3p1d-dep4-dep16.yaml +++ /dev/null @@ -1,99 +0,0 @@ -name: "kimi-vllm-disagg-gb200-3p1d-dep4-dep16" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.2.1 - install: true - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 4 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 64 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.93 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 256 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - all2all-backend: "flashinfer_nvlink_one_sided" - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 256 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1024" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-6p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-6p1d-dep4-dep16.yaml deleted file mode 100644 index 1f4de8df10..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-6p1d-dep4-dep16.yaml +++ /dev/null @@ -1,99 +0,0 @@ -name: "kimi-vllm-disagg-gb200-6p1d-dep4-dep16" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.2.1 - install: true - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 6 - decode_nodes: 4 - prefill_workers: 6 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 64 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.93 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 512 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - all2all-backend: "flashinfer_nvlink_one_sided" - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 512 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "3072" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-8p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-8p1d-dep4-dep16.yaml deleted file mode 100644 index 45a7214b2d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-8p1d-dep4-dep16.yaml +++ /dev/null @@ -1,100 +0,0 @@ -name: "kimi-vllm-disagg-gb200-8p1d-dep4-dep16" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.2.1 - install: true - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 8 - decode_nodes: 4 - prefill_workers: 8 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 64 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.93 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 256 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - all2all-backend: "flashinfer_nvlink_one_sided" - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 512 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "6144" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-1p4d-dep4-tp8.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-1p4d-dep4-tp8.yaml deleted file mode 100644 index 0ed5673478..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-1p4d-dep4-tp8.yaml +++ /dev/null @@ -1,96 +0,0 @@ -name: "kimi-vllm-disagg-gb300-1p4d-dep4-tp8" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.3.0.dev20260601 - install: true - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 8 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 4096 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - block-size: 64 - attention-backend: "FLASHINFER_MLA" - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.9 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - max-model-len: 10240 - max-num-seqs: 4 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - all2all-backend: "flashinfer_nvlink_one_sided" - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 1024 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-1p8d-dep4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-1p8d-dep4-tp4.yaml deleted file mode 100644 index 71d897b97d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-1p8d-dep4-tp4.yaml +++ /dev/null @@ -1,96 +0,0 @@ -name: "kimi-vllm-disagg-gb300-1p8d-dep4-tp4" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.3.0.dev20260601 - install: true - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 8 - prefill_workers: 1 - decode_workers: 8 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 4096 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - block-size: 64 - attention-backend: "FLASHINFER_MLA" - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.9 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - max-model-len: 10240 - max-num-seqs: 768 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - all2all-backend: "flashinfer_nvlink_one_sided" - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 1024 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "32x128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-2p1d-dep4-dep24.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-2p1d-dep4-dep24.yaml deleted file mode 100644 index b43b9f1367..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-2p1d-dep4-dep24.yaml +++ /dev/null @@ -1,99 +0,0 @@ -name: "kimi-vllm-disagg-gb300-2p1d-dep4-dep24" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.3.0.dev20260601 - install: true - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 6 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 24 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 4096 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - block-size: 64 - attention-backend: "FLASHINFER_MLA" - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.9 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 24 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 43 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - all2all-backend: "flashinfer_nvlink_one_sided" - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 128 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-3p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-3p1d-dep4-dep16.yaml deleted file mode 100644 index 7ee94c9d10..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-3p1d-dep4-dep16.yaml +++ /dev/null @@ -1,99 +0,0 @@ -name: "kimi-vllm-disagg-gb300-3p1d-dep4-dep16" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.3.0.dev20260601 - install: true - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 4 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 4096 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - block-size: 64 - attention-backend: "FLASHINFER_MLA" - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.9 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 64 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - all2all-backend: "flashinfer_nvlink_one_sided" - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 128 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-4p1d-dep4-dep8.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-4p1d-dep4-dep8.yaml deleted file mode 100644 index 23fc7f67f0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-4p1d-dep4-dep8.yaml +++ /dev/null @@ -1,100 +0,0 @@ -name: "kimi-vllm-disagg-gb300-4p1d-dep4-dep8" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.3.0.dev20260601 - install: true - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 2 - prefill_workers: 4 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 4096 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - block-size: 64 - attention-backend: "FLASHINFER_MLA" - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.9 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 256 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - all2all-backend: "flashinfer_nvlink_one_sided" - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 512 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "3072" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-8p1d-dep4-dep24.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-8p1d-dep4-dep24.yaml deleted file mode 100644 index 8b2f2bf80c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-8p1d-dep4-dep24.yaml +++ /dev/null @@ -1,100 +0,0 @@ -name: "kimi-vllm-disagg-gb300-8p1d-dep4-dep24" - -model: - path: "kimi-k2.5-nvfp4" - container: "vllm/vllm-openai:v0.21.0" - precision: "fp4" - -dynamo: - version: 1.3.0.dev20260601 - install: true - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 8 - decode_nodes: 6 - prefill_workers: 8 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 24 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_USE_FLASHINFER_MOE_FP4: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 4096 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - block-size: 64 - attention-backend: "FLASHINFER_MLA" - attention-config: '{"use_trtllm_ragged_deepseek_prefill": true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.9 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "nvidia/Kimi-K2.5-NVFP4" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 24 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 512 - max-num-batched-tokens: 10240 - safetensors-load-strategy: "prefetch" - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-chunked-prefill: true - async-scheduling: true - attention-backend: "FLASHINFER_MLA" - block-size: 64 - all2all-backend: "flashinfer_nvlink_one_sided" - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 128 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "15360" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml deleted file mode 100644 index b7378fe35f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml +++ /dev/null @@ -1,125 +0,0 @@ -name: kimi-k2.6-vllm-disagg-b200-1p1d-dep8-dep8-c2048 -model: - path: kimi-k2.6-nvfp4 - container: vllm/vllm-openai:v0.25.1 - precision: fp4 -dynamo: - wheel: 1.3.0.dev20260721 - install: true -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 -infra: - etcd_nats_dedicated_node: true -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - dp_launch_mode: per_gpu - prefill_environment: - PYTHONUNBUFFERED: '1' - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_NCCL_SYMM_MEM: '0' - NCCL_CUMEM_ENABLE: '1' - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' - NCCL_WATCHDOG_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_CUDA_IPC_ENABLE_MNNVL: n - UCX_MAX_RMA_RAILS: '1' - UCX_MAX_RNDV_RAILS: '1' - UCX_RNDV_SCHEME: put_zcopy - decode_environment: - PYTHONUNBUFFERED: '1' - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_NCCL_SYMM_MEM: '0' - NCCL_CUMEM_ENABLE: '1' - NCCL_WATCHDOG_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_CUDA_IPC_ENABLE_MNNVL: n - UCX_MAX_RMA_RAILS: '1' - UCX_MAX_RNDV_RAILS: '1' - UCX_RNDV_SCHEME: put_zcopy - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep8"}' - served-model-name: nvidia/Kimi-K2.6-NVFP4 - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13346 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 2048 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: prefetch - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - attention-backend: FLASHINFER_MLA - block-size: 128 - attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' - gpu-memory-utilization: 0.9 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: nvidia/Kimi-K2.6-NVFP4 - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 2048 - max-num-batched-tokens: 8192 - safetensors-load-strategy: prefetch - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - async-scheduling: true - attention-backend: FLASHINFER_MLA - block-size: 128 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 1024 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '2048' - req_rate: inf - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml deleted file mode 100644 index cf8d37ff59..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml +++ /dev/null @@ -1,123 +0,0 @@ -name: kimi-k2.6-vllm-disagg-b200-1p1d-dep8-tp8-c1 -model: - path: kimi-k2.6-nvfp4 - container: vllm/vllm-openai:v0.25.1 - precision: fp4 -dynamo: - wheel: 1.3.0.dev20260721 - install: true -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 -infra: - etcd_nats_dedicated_node: true -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - dp_launch_mode: per_gpu - prefill_environment: - PYTHONUNBUFFERED: '1' - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_NCCL_SYMM_MEM: '0' - NCCL_CUMEM_ENABLE: '1' - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' - NCCL_WATCHDOG_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_CUDA_IPC_ENABLE_MNNVL: n - UCX_MAX_RMA_RAILS: '1' - UCX_MAX_RNDV_RAILS: '1' - UCX_RNDV_SCHEME: put_zcopy - decode_environment: - PYTHONUNBUFFERED: '1' - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_NCCL_SYMM_MEM: '0' - NCCL_CUMEM_ENABLE: '0' - NCCL_WATCHDOG_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_CUDA_IPC_ENABLE_MNNVL: n - UCX_MAX_RMA_RAILS: '1' - UCX_MAX_RNDV_RAILS: '1' - UCX_RNDV_SCHEME: put_zcopy - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep8"}' - served-model-name: nvidia/Kimi-K2.6-NVFP4 - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13346 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 2048 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: prefetch - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - attention-backend: FLASHINFER_MLA - block-size: 128 - attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' - gpu-memory-utilization: 0.9 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: nvidia/Kimi-K2.6-NVFP4 - kv-cache-dtype: fp8 - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - max-model-len: 9216 - max-num-seqs: 1024 - max-num-batched-tokens: 8192 - safetensors-load-strategy: prefetch - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - async-scheduling: true - attention-backend: FLASHINFER_MLA - block-size: 128 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 256 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '1' - req_rate: inf - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml deleted file mode 100644 index f17b45287c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml +++ /dev/null @@ -1,122 +0,0 @@ -name: kimi-k2.6-vllm-disagg-b200-1p4d-dep4-tp4-c512 -model: - path: kimi-k2.6-nvfp4 - container: vllm/vllm-openai:v0.25.1 - precision: fp4 -dynamo: - wheel: 1.3.0.dev20260721 - install: true -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 4 - gpus_per_decode: 4 -infra: - etcd_nats_dedicated_node: true -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - dp_launch_mode: per_gpu - prefill_environment: - PYTHONUNBUFFERED: '1' - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_NCCL_SYMM_MEM: '0' - NCCL_CUMEM_ENABLE: '1' - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' - NCCL_WATCHDOG_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_CUDA_IPC_ENABLE_MNNVL: n - UCX_MAX_RMA_RAILS: '1' - UCX_MAX_RNDV_RAILS: '1' - UCX_RNDV_SCHEME: put_zcopy - decode_environment: - PYTHONUNBUFFERED: '1' - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_NCCL_SYMM_MEM: '0' - NCCL_CUMEM_ENABLE: '1' - NCCL_WATCHDOG_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_CUDA_IPC_ENABLE_MNNVL: n - UCX_MAX_RMA_RAILS: '1' - UCX_MAX_RNDV_RAILS: '1' - UCX_RNDV_SCHEME: put_zcopy - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' - served-model-name: nvidia/Kimi-K2.6-NVFP4 - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13346 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 2048 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 8192 - safetensors-load-strategy: prefetch - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - attention-backend: FLASHINFER_MLA - block-size: 128 - attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' - gpu-memory-utilization: 0.94 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: nvidia/Kimi-K2.6-NVFP4 - kv-cache-dtype: fp8 - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - max-model-len: 9216 - max-num-seqs: 2048 - max-num-batched-tokens: 8192 - safetensors-load-strategy: prefetch - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - async-scheduling: true - attention-backend: FLASHINFER_MLA - block-size: 128 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.93 - stream-interval: 50 - max-cudagraph-capture-size: 2048 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '512' - req_rate: inf - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml deleted file mode 100644 index e33c6b773f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml +++ /dev/null @@ -1,122 +0,0 @@ -name: kimi-k2.6-vllm-disagg-b200-1p8d-dep4-tp4-c128 -model: - path: kimi-k2.6-nvfp4 - container: vllm/vllm-openai:v0.25.1 - precision: fp4 -dynamo: - wheel: 1.3.0.dev20260721 - install: true -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 8 - gpus_per_prefill: 4 - gpus_per_decode: 4 -infra: - etcd_nats_dedicated_node: true -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - dp_launch_mode: per_gpu - prefill_environment: - PYTHONUNBUFFERED: '1' - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_NCCL_SYMM_MEM: '0' - NCCL_CUMEM_ENABLE: '1' - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' - NCCL_WATCHDOG_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_CUDA_IPC_ENABLE_MNNVL: n - UCX_MAX_RMA_RAILS: '1' - UCX_MAX_RNDV_RAILS: '1' - UCX_RNDV_SCHEME: put_zcopy - decode_environment: - PYTHONUNBUFFERED: '1' - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_NCCL_SYMM_MEM: '0' - NCCL_CUMEM_ENABLE: '1' - NCCL_WATCHDOG_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_CUDA_IPC_ENABLE_MNNVL: n - UCX_MAX_RMA_RAILS: '1' - UCX_MAX_RNDV_RAILS: '1' - UCX_RNDV_SCHEME: put_zcopy - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' - served-model-name: nvidia/Kimi-K2.6-NVFP4 - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13346 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 2048 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 8192 - safetensors-load-strategy: prefetch - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - attention-backend: FLASHINFER_MLA - block-size: 128 - attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' - gpu-memory-utilization: 0.94 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: nvidia/Kimi-K2.6-NVFP4 - kv-cache-dtype: fp8 - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - max-model-len: 9216 - max-num-seqs: 2048 - max-num-batched-tokens: 8192 - safetensors-load-strategy: prefetch - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - async-scheduling: true - attention-backend: FLASHINFER_MLA - block-size: 128 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.93 - stream-interval: 50 - max-cudagraph-capture-size: 2048 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '128' - req_rate: inf - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml deleted file mode 100644 index 07f3c13792..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml +++ /dev/null @@ -1,122 +0,0 @@ -name: kimi-k2.6-vllm-disagg-b200-1p8d-dep4-tp4-c32 -model: - path: kimi-k2.6-nvfp4 - container: vllm/vllm-openai:v0.25.1 - precision: fp4 -dynamo: - wheel: 1.3.0.dev20260721 - install: true -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 8 - gpus_per_prefill: 4 - gpus_per_decode: 4 -infra: - etcd_nats_dedicated_node: true -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - dp_launch_mode: per_gpu - prefill_environment: - PYTHONUNBUFFERED: '1' - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_NCCL_SYMM_MEM: '0' - NCCL_CUMEM_ENABLE: '1' - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' - NCCL_WATCHDOG_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_CUDA_IPC_ENABLE_MNNVL: n - UCX_MAX_RMA_RAILS: '1' - UCX_MAX_RNDV_RAILS: '1' - UCX_RNDV_SCHEME: put_zcopy - decode_environment: - PYTHONUNBUFFERED: '1' - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_NCCL_SYMM_MEM: '0' - NCCL_CUMEM_ENABLE: '1' - NCCL_WATCHDOG_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_CUDA_IPC_ENABLE_MNNVL: n - UCX_MAX_RMA_RAILS: '1' - UCX_MAX_RNDV_RAILS: '1' - UCX_RNDV_SCHEME: put_zcopy - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' - served-model-name: nvidia/Kimi-K2.6-NVFP4 - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13346 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 2048 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 8192 - safetensors-load-strategy: prefetch - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - attention-backend: FLASHINFER_MLA - block-size: 128 - attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' - gpu-memory-utilization: 0.93 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: nvidia/Kimi-K2.6-NVFP4 - kv-cache-dtype: fp8 - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - max-model-len: 9216 - max-num-seqs: 2048 - max-num-batched-tokens: 8192 - safetensors-load-strategy: prefetch - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - async-scheduling: true - attention-backend: FLASHINFER_MLA - block-size: 128 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.93 - stream-interval: 50 - max-cudagraph-capture-size: 2048 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '32' - req_rate: inf - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml deleted file mode 100644 index 9c80baf673..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml +++ /dev/null @@ -1,125 +0,0 @@ -name: kimi-k2.6-vllm-disagg-b200-2p1d-dep8-dep8-c8192 -model: - path: kimi-k2.6-nvfp4 - container: vllm/vllm-openai:v0.25.1 - precision: fp4 -dynamo: - wheel: 1.3.0.dev20260721 - install: true -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 2 - decode_nodes: 1 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 -infra: - etcd_nats_dedicated_node: true -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - dp_launch_mode: per_gpu - prefill_environment: - PYTHONUNBUFFERED: '1' - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_NCCL_SYMM_MEM: '0' - NCCL_CUMEM_ENABLE: '1' - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' - NCCL_WATCHDOG_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_CUDA_IPC_ENABLE_MNNVL: n - UCX_MAX_RMA_RAILS: '1' - UCX_MAX_RNDV_RAILS: '1' - UCX_RNDV_SCHEME: put_zcopy - decode_environment: - PYTHONUNBUFFERED: '1' - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_NCCL_SYMM_MEM: '0' - NCCL_CUMEM_ENABLE: '1' - NCCL_WATCHDOG_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_CUDA_IPC_ENABLE_MNNVL: n - UCX_MAX_RMA_RAILS: '1' - UCX_MAX_RNDV_RAILS: '1' - UCX_RNDV_SCHEME: put_zcopy - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: nvidia/Kimi-K2.6-NVFP4 - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13346 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 2048 - enforce-eager: true - compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - max-num-batched-tokens: 16384 - safetensors-load-strategy: prefetch - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - attention-backend: FLASHINFER_MLA - block-size: 128 - attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' - gpu-memory-utilization: 0.9 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: nvidia/Kimi-K2.6-NVFP4 - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 10240 - max-num-seqs: 2048 - max-num-batched-tokens: 8192 - safetensors-load-strategy: prefetch - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - async-scheduling: true - attention-backend: FLASHINFER_MLA - block-size: 128 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - max-cudagraph-capture-size: 1024 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: '8192' - req_rate: inf - use_chat_template: true - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 10 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c1-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c1-agentic.yaml deleted file mode 100644 index 5510b41124..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c1-agentic.yaml +++ /dev/null @@ -1,142 +0,0 @@ -name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c1-agentic" - -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - MC_GID_INDEX: "3" - MC_STORE_MEMCPY: "1" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_SLICE_SIZE: "1048576" - MC_WORKERS_PER_CTX: "4" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_RPC_TIMEOUT: "600000" - WITH_NVIDIA_PEERMEM: "0" - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - decode-context-parallel-size: 8 - dcp-comm-backend: a2a - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - gpu-memory-utilization: 0.92 - no-enable-flashinfer-autotune: true - max-model-len: 1048576 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' - attention-backend: TOKENSPEED_MLA - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - enable-prefix-caching: true - enable-prompt-tokens-details: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - max-num-seqs: 2 - max-num-batched-tokens: 8192 - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' - prefix-match-unit: 128 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,64,128,256,512,1024,2048,4096,8192]}' - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - -benchmark: - type: custom - concurrencies: [1] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c14-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c14-agentic.yaml deleted file mode 100644 index a4f0f4eb45..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c14-agentic.yaml +++ /dev/null @@ -1,142 +0,0 @@ -name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c14-agentic" - -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - MC_GID_INDEX: "3" - MC_STORE_MEMCPY: "1" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_SLICE_SIZE: "1048576" - MC_WORKERS_PER_CTX: "4" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_RPC_TIMEOUT: "600000" - WITH_NVIDIA_PEERMEM: "0" - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - decode-context-parallel-size: 8 - dcp-comm-backend: a2a - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - gpu-memory-utilization: 0.92 - no-enable-flashinfer-autotune: true - max-model-len: 1048576 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' - attention-backend: TOKENSPEED_MLA - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - enable-prefix-caching: true - enable-prompt-tokens-details: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - max-num-seqs: 28 - max-num-batched-tokens: 8192 - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' - prefix-match-unit: 128 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,40,56,64,80,104,128,256,512,1024,2048,4096,8192]}' - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - -benchmark: - type: custom - concurrencies: [14] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c24-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c24-agentic.yaml deleted file mode 100644 index 6dab857310..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c24-agentic.yaml +++ /dev/null @@ -1,142 +0,0 @@ -name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c24-agentic" - -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - MC_GID_INDEX: "3" - MC_STORE_MEMCPY: "1" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_SLICE_SIZE: "1048576" - MC_WORKERS_PER_CTX: "4" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_RPC_TIMEOUT: "600000" - WITH_NVIDIA_PEERMEM: "0" - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - decode-context-parallel-size: 8 - dcp-comm-backend: a2a - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - gpu-memory-utilization: 0.92 - no-enable-flashinfer-autotune: true - max-model-len: 1048576 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' - attention-backend: TOKENSPEED_MLA - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - enable-prefix-caching: true - enable-prompt-tokens-details: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - max-num-seqs: 48 - max-num-batched-tokens: 8192 - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' - prefix-match-unit: 128 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,40,56,64,80,104,128,256,512,1024,2048,4096,8192]}' - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - -benchmark: - type: custom - concurrencies: [24] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c4-agentic.yaml deleted file mode 100644 index 1769cd164a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c4-agentic.yaml +++ /dev/null @@ -1,142 +0,0 @@ -name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c4-agentic" - -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - MC_GID_INDEX: "3" - MC_STORE_MEMCPY: "1" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_SLICE_SIZE: "1048576" - MC_WORKERS_PER_CTX: "4" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_RPC_TIMEOUT: "600000" - WITH_NVIDIA_PEERMEM: "0" - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - decode-context-parallel-size: 8 - dcp-comm-backend: a2a - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - gpu-memory-utilization: 0.92 - no-enable-flashinfer-autotune: true - max-model-len: 1048576 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' - attention-backend: TOKENSPEED_MLA - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - enable-prefix-caching: true - enable-prompt-tokens-details: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - max-num-seqs: 8 - max-num-batched-tokens: 8192 - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' - prefix-match-unit: 128 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,128,256,512,1024,2048,4096,8192]}' - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - -benchmark: - type: custom - concurrencies: [4] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c48-agentic.yaml deleted file mode 100644 index 3c64e5a12d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c48-agentic.yaml +++ /dev/null @@ -1,142 +0,0 @@ -name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c48-agentic" - -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - MC_GID_INDEX: "3" - MC_STORE_MEMCPY: "1" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_SLICE_SIZE: "1048576" - MC_WORKERS_PER_CTX: "4" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_RPC_TIMEOUT: "600000" - WITH_NVIDIA_PEERMEM: "0" - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - decode-context-parallel-size: 8 - dcp-comm-backend: a2a - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - gpu-memory-utilization: 0.92 - no-enable-flashinfer-autotune: true - max-model-len: 1048576 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' - attention-backend: TOKENSPEED_MLA - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - enable-prefix-caching: true - enable-prompt-tokens-details: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - max-num-seqs: 96 - max-num-batched-tokens: 8192 - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":4,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' - prefix-match-unit: 128 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,7,18,34,53,64,75,100,128,256,512,1024,2048,4096,8192]}' - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - -benchmark: - type: custom - concurrencies: [48] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c8-agentic.yaml deleted file mode 100644 index 390395736a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c8-agentic.yaml +++ /dev/null @@ -1,142 +0,0 @@ -name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c8-agentic" - -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - MC_GID_INDEX: "3" - MC_STORE_MEMCPY: "1" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_SLICE_SIZE: "1048576" - MC_WORKERS_PER_CTX: "4" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_RPC_TIMEOUT: "600000" - WITH_NVIDIA_PEERMEM: "0" - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - decode-context-parallel-size: 8 - dcp-comm-backend: a2a - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - gpu-memory-utilization: 0.92 - no-enable-flashinfer-autotune: true - max-model-len: 1048576 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' - attention-backend: TOKENSPEED_MLA - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - enable-prefix-caching: true - enable-prompt-tokens-details: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - max-num-seqs: 16 - max-num-batched-tokens: 8192 - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' - prefix-match-unit: 128 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,40,56,64,80,104,128,256,512,1024,2048,4096,8192]}' - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - -benchmark: - type: custom - concurrencies: [8] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c96-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c96-agentic.yaml deleted file mode 100644 index 8b4b2d6491..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c96-agentic.yaml +++ /dev/null @@ -1,141 +0,0 @@ -name: "kimik3-vllm-agg-b200-tp8pp2-mooncake-c96-agentic" - -model: - path: "kimik3" - container: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-728d3ad" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - MC_GID_INDEX: "3" - MC_STORE_MEMCPY: "1" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_SLICE_SIZE: "1048576" - MC_WORKERS_PER_CTX: "4" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_RPC_TIMEOUT: "600000" - WITH_NVIDIA_PEERMEM: "0" - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - decode-context-parallel-size: 8 - dcp-comm-backend: a2a - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - gpu-memory-utilization: 0.92 - no-enable-flashinfer-autotune: true - max-model-len: 1048576 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' - attention-backend: TOKENSPEED_MLA - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - enable-prefix-caching: true - enable-prompt-tokens-details: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - max-num-seqs: 192 - max-num-batched-tokens: 8192 - prefix-match-unit: 128 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,7,18,34,53,64,75,100,128,256,512,1024,2048,4096,8192]}' - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - -benchmark: - type: custom - concurrencies: [96] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-dspark4-maxseq2-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-dspark4-maxseq2-mooncake-agentic.yaml deleted file mode 100644 index 6878c47b00..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-dspark4-maxseq2-mooncake-agentic.yaml +++ /dev/null @@ -1,148 +0,0 @@ -# GB200 TP16/DCP16 aggregate profile with DSpark K=4 and max-num-seqs 2. -name: "kimi-k3-vllm-agg-gb200-dcp16-dspark4-maxseq2-mooncake-agentic" - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" - frameworks: - dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -slurm: - time_limit: "04:00:00" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 4 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - enable-auto-tool-choice: true - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - -backend: - type: vllm - connector: null - mooncake_kv_store: - master_extra_args: - - "--eviction_high_watermark_ratio=0.95" - - "--eviction_ratio=0.10" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "96GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - # Preserve the measured collective fallback when moving to the built image. - VLLM_USE_DIRECT_DCP_A2A: "0" - VLLM_USE_DIRECT_DCP_Q_GATHER: "0" - VLLM_USE_DIRECT_DCP_KV_GATHER: "0" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" - PYTHONNOUSERSITE: "1" - TORCH_CUDA_ARCH_LIST: "10.0" - PYTHONHASHSEED: "42" - VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" - MC_GID_INDEX: "3" - MC_STORE_MEMCPY: "1" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_SLICE_SIZE: "1048576" - MC_WORKERS_PER_CTX: "4" - WITH_NVIDIA_PEERMEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 - decode-context-parallel-size: 16 - dcp-comm-backend: "a2a" - max-num-seqs: 2 - max-num-batched-tokens: 8192 - trust-remote-code: true - language-model-only: true - dyn-tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - dyn-reasoning-parser: "kimi_k3" - load-format: "safetensors" - safetensors-load-strategy: "lazy" - moe-backend: "auto" - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - enable-prefix-caching: true - prefix-match-unit: 128 - kv-cache-dtype: "fp8" - stream-interval: 10 - attention-backend: "TOKENSPEED_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - # Throughput jobs inject the committed K=4 golden AL (3.36); EVAL_ONLY - # preserves this real target-verification configuration. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' - max-cudagraph-capture-size: 1024 - kv-cache-memory: 10737418240 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - comment: >- - '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-nospec-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-nospec-mooncake-agentic.yaml deleted file mode 100644 index e10ec63b24..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-nospec-mooncake-agentic.yaml +++ /dev/null @@ -1,144 +0,0 @@ -# GB200 TP16/DCP16 aggregate profile without speculative decoding. -name: "kimi-k3-vllm-agg-gb200-dcp16-nospec-mooncake-agentic" - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" - frameworks: - dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -slurm: - time_limit: "04:00:00" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 4 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - enable-auto-tool-choice: true - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - -backend: - type: vllm - connector: null - mooncake_kv_store: - master_extra_args: - - "--eviction_high_watermark_ratio=0.95" - - "--eviction_ratio=0.10" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "96GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - # Preserve the measured collective fallback when moving to the built image. - VLLM_USE_DIRECT_DCP_A2A: "0" - VLLM_USE_DIRECT_DCP_Q_GATHER: "0" - VLLM_USE_DIRECT_DCP_KV_GATHER: "0" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" - PYTHONNOUSERSITE: "1" - TORCH_CUDA_ARCH_LIST: "10.0" - PYTHONHASHSEED: "42" - VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" - MC_GID_INDEX: "3" - MC_STORE_MEMCPY: "1" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_SLICE_SIZE: "1048576" - MC_WORKERS_PER_CTX: "4" - WITH_NVIDIA_PEERMEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 - decode-context-parallel-size: 16 - dcp-comm-backend: "a2a" - max-num-batched-tokens: 16384 - trust-remote-code: true - language-model-only: true - dyn-tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - dyn-reasoning-parser: "kimi_k3" - load-format: "safetensors" - safetensors-load-strategy: "lazy" - moe-backend: "auto" - no-enable-flashinfer-autotune: true - enable-prefix-caching: true - prefix-match-unit: 128 - kv-cache-dtype: "fp8" - stream-interval: 10 - attention-backend: "FLASHINFER_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - max-cudagraph-capture-size: 1024 - max-num-seqs: 1000 - kv-cache-memory: 10737418240 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - comment: >- - '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml deleted file mode 100644 index f1fde0a104..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml +++ /dev/null @@ -1,150 +0,0 @@ -name: "kimi-k3-vllm-agg-gb200-dep16-throughput-agentic" - -# Day-0 GB200 translation of the official throughput-oriented multi_node_dep -# profile. TP4 x DP4 gives EP16 across four four-GPU GB200 nodes. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_dep - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.3.0" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -setup_script: kimik3-dspark-config-compat.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "12:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 4 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - # Use vLLM's Kimi K3 parser so OpenAI responses expose structured - # tool_calls instead of raw XTML in message.content. - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-min-initial-workers: 1 - kv-cache-block-size: 64 - -backend: - type: vllm - connector: null - aggregated_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - NVIDIA_GDRCOPY: "1" - PYTORCH_ALLOC_CONF: "expandable_segments:True" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-dep16-{job_id}" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - # FlashInfer's larger TP4 MoE representation leaves too little transient - # HBM for fastsafetensors' GPU-staging path when DSpark is loaded. - load-format: "safetensors" - safetensors-load-strategy: "lazy" - kv-cache-dtype: "fp8" - attention-backend: "FLASHINFER_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - # DeepGEMM mega-MoE grid barriers time out under DSpark MRV2 with the - # one-sided all-to-all worker; use the supported Kimi K3 FlashInfer path. - moe-backend: "flashinfer_trtllm" - kda-prefill-backend: "flashkda" - kernel-config: '{"enable_cutedsl_warmup":true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.94 - # The largest regular DEP point is c256 / DP4 = 64 sequences per engine. - # Capturing 128 sequence slots consumes 10.9 GiB and leaves too little - # runtime workspace for FlashInfer's MXFP4 MoE kernel. - max-num-seqs: 64 - max-num-batched-tokens: 16384 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' - compilation-config: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96,99,102,105,108,111,114,117,120,123,126,129,132,135,138,141,144,147,150,153,156,159,162,165,168,171,174,177,180,183,186,189,192]}' - block-size: 64 - language-model-only: true - disable-custom-all-reduce: true - enable-prefix-caching: true - scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" - dyn-tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - dyn-reasoning-parser: "kimi_k3" - no-enable-flashinfer-autotune: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AGENTIC_WARMUP_GRACE_PERIOD: "3600" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml deleted file mode 100644 index b748d10f21..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml +++ /dev/null @@ -1,153 +0,0 @@ -name: "kimi-k3-vllm-agg-gb200-dep16-throughput-vllm-simple-offload-agentic" - -# High-concurrency host-DRAM KV-offload variant of the official throughput- -# oriented multi_node_dep profile. TP4 x DP4 gives EP16 across four four-GPU -# GB200 nodes. Each TP rank receives a 128 GiB CPU KV pool (512 GiB per node). -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_dep - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.3.0" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -setup_script: kimik3-dspark-config-compat.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "12:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 4 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - # Use vLLM's Kimi K3 parser so OpenAI responses expose structured - # tool_calls instead of raw XTML in message.content. - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-min-initial-workers: 1 - kv-cache-block-size: 64 - -backend: - type: vllm - connector: null - aggregated_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - NVIDIA_GDRCOPY: "1" - PYTHONHASHSEED: "42" - PYTORCH_ALLOC_CONF: "expandable_segments:True" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-dep16-offload-{job_id}" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - # FlashInfer's larger TP4 MoE representation leaves too little transient - # HBM for fastsafetensors' GPU-staging path when DSpark is loaded. - load-format: "safetensors" - safetensors-load-strategy: "lazy" - kv-cache-dtype: "fp8" - attention-backend: "FLASHINFER_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - # DeepGEMM mega-MoE grid barriers time out under DSpark MRV2 with the - # one-sided all-to-all worker; use the supported Kimi K3 FlashInfer path. - moe-backend: "flashinfer_trtllm" - kda-prefill-backend: "flashkda" - kernel-config: '{"enable_cutedsl_warmup":true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.94 - # The largest offload point is c384 / DP4 = 96 sequences per engine. - # Capture even sequence counts: all configured DP4 steady-state batch - # sizes are exact hits, while odd loads pad by at most one sequence. - max-num-seqs: 96 - max-num-batched-tokens: 16384 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' - compilation-config: '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96,102,108,114,120,126,132,138,144,150,156,162,168,174,180,186,192,198,204,210,216,222,228,234,240,246,252,258,264,270,276,282,288]}' - block-size: 64 - language-model-only: true - disable-custom-all-reduce: true - enable-prefix-caching: true - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' - scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" - dyn-tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - dyn-reasoning-parser: "kimi_k3" - no-enable-flashinfer-autotune: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AGENTIC_WARMUP_GRACE_PERIOD: "3600" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml deleted file mode 100644 index be797ce972..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml +++ /dev/null @@ -1,141 +0,0 @@ -name: "kimi-k3-vllm-agg-gb200-tep16-balanced-agentic" - -# Day-0 GB200 translation of the official balanced multi_node_tep profile. -# Dense layers and MoE experts are sharded across 16 GPUs on four GB200 nodes -# with the official FP8 KV cache. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tep - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.3.0" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -setup_script: kimik3-dspark-config-compat.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 4 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - # Use vLLM's Kimi K3 parser so OpenAI responses expose structured - # tool_calls instead of raw XTML in message.content. - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-min-initial-workers: 1 - kv-cache-block-size: 64 - -backend: - type: vllm - connector: null - aggregated_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - NVIDIA_GDRCOPY: "1" - PYTORCH_ALLOC_CONF: "expandable_segments:True" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-tep16-{job_id}" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 - pipeline-parallel-size: 1 - enable-expert-parallel: true - trust-remote-code: true - load-format: "fastsafetensors" - safetensors-load-strategy: "lazy" - kv-cache-dtype: "fp8" - attention-backend: "FLASHINFER_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - moe-backend: "flashinfer_trtllm" - kda-prefill-backend: "flashkda" - kernel-config: '{"enable_cutedsl_warmup":true}' - all2all-backend: "flashinfer_nvlink_one_sided" - gpu-memory-utilization: 0.92 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' - compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96],"pass_config":{"fuse_allreduce_rms":false}}' - block-size: 64 - language-model-only: true - disable-custom-all-reduce: true - enable-prefix-caching: true - scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" - dyn-tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - dyn-reasoning-parser: "kimi_k3" - no-enable-flashinfer-autotune: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml deleted file mode 100644 index 221ab1693e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml +++ /dev/null @@ -1,139 +0,0 @@ -name: "kimi-k3-vllm-agg-gb200-tp16-latency-agentic" - -# Day-0 GB200 translation of the official latency-oriented multi_node_tp -# profile. TP16 spans four GB200 nodes and uses the official FP8 KV cache. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tp - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.3.0" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -setup_script: kimik3-dspark-config-compat.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 4 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - # Use vLLM's Kimi K3 parser so OpenAI responses expose structured - # tool_calls instead of raw XTML in message.content. - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-min-initial-workers: 1 - kv-cache-block-size: 64 - -backend: - type: vllm - connector: null - aggregated_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - NVIDIA_GDRCOPY: "1" - PYTORCH_ALLOC_CONF: "expandable_segments:True" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-kimi-k3-gb200-tp16-{job_id}" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 - pipeline-parallel-size: 1 - trust-remote-code: true - load-format: "fastsafetensors" - safetensors-load-strategy: "lazy" - kv-cache-dtype: "fp8" - attention-backend: "FLASHINFER_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - moe-backend: "flashinfer_trtllm" - kda-prefill-backend: "flashkda" - kernel-config: '{"enable_cutedsl_warmup":true}' - gpu-memory-utilization: 0.92 - max-num-seqs: 8 - max-num-batched-tokens: 8192 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' - compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24],"pass_config":{"fuse_allreduce_rms":false}}' - block-size: 64 - language-model-only: true - disable-custom-all-reduce: true - enable-prefix-caching: true - scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" - dyn-tool-call-parser: "kimi_k3" - reasoning-parser: "kimi_k3" - dyn-reasoning-parser: "kimi_k3" - no-enable-flashinfer-autotune: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c16-agentic.yaml deleted file mode 100644 index 706d461202..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c16-agentic.yaml +++ /dev/null @@ -1,129 +0,0 @@ -name: "kimik3-vllm-agg-gb200-tp8pp2-mooncake-c16-agentic" - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 4 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - MC_GID_INDEX: "3" - MC_STORE_MEMCPY: "1" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_SLICE_SIZE: "1048576" - MC_WORKERS_PER_CTX: "4" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_RPC_TIMEOUT: "600000" - WITH_NVIDIA_PEERMEM: "0" - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - decode-context-parallel-size: 8 - dcp-comm-backend: a2a - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - gpu-memory-utilization: 0.92 - no-enable-flashinfer-autotune: true - max-model-len: 1048576 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' - attention-backend: TOKENSPEED_MLA - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - enable-prefix-caching: true - enable-prompt-tokens-details: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - max-num-seqs: 32 - max-num-batched-tokens: 8192 - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' - prefix-match-unit: 128 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,40,56,64,80,104,128,256,512,1024,2048,4096,8192]}' - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c32-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c32-agentic.yaml deleted file mode 100644 index c380657c35..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c32-agentic.yaml +++ /dev/null @@ -1,129 +0,0 @@ -name: "kimik3-vllm-agg-gb200-tp8pp2-mooncake-c32-agentic" - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 4 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - MC_GID_INDEX: "3" - MC_STORE_MEMCPY: "1" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_SLICE_SIZE: "1048576" - MC_WORKERS_PER_CTX: "4" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_RPC_TIMEOUT: "600000" - WITH_NVIDIA_PEERMEM: "0" - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - decode-context-parallel-size: 8 - dcp-comm-backend: a2a - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - gpu-memory-utilization: 0.92 - no-enable-flashinfer-autotune: true - max-model-len: 1048576 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' - attention-backend: TOKENSPEED_MLA - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - enable-prefix-caching: true - enable-prompt-tokens-details: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - max-num-seqs: 64 - max-num-batched-tokens: 8192 - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' - prefix-match-unit: 128 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,7,18,34,53,64,75,100,128,256,512,1024,2048,4096,8192]}' - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c48-agentic.yaml deleted file mode 100644 index c5be318803..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c48-agentic.yaml +++ /dev/null @@ -1,128 +0,0 @@ -name: "kimik3-vllm-agg-gb200-tp8pp2-mooncake-c48-agentic" - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 4 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - MC_GID_INDEX: "3" - MC_STORE_MEMCPY: "1" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_SLICE_SIZE: "1048576" - MC_WORKERS_PER_CTX: "4" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_RPC_TIMEOUT: "600000" - WITH_NVIDIA_PEERMEM: "0" - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - decode-context-parallel-size: 8 - dcp-comm-backend: a2a - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - gpu-memory-utilization: 0.92 - no-enable-flashinfer-autotune: true - max-model-len: 1048576 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' - attention-backend: TOKENSPEED_MLA - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - enable-prefix-caching: true - enable-prompt-tokens-details: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - max-num-seqs: 96 - max-num-batched-tokens: 8192 - prefix-match-unit: 128 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,7,18,34,53,64,75,100,128,256,512,1024,2048,4096,8192]}' - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c72-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c72-agentic.yaml deleted file mode 100644 index bac97e2781..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c72-agentic.yaml +++ /dev/null @@ -1,128 +0,0 @@ -name: "kimik3-vllm-agg-gb200-tp8pp2-mooncake-c72-agentic" - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 4 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - MC_GID_INDEX: "3" - MC_STORE_MEMCPY: "1" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_SLICE_SIZE: "1048576" - MC_WORKERS_PER_CTX: "4" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_RPC_TIMEOUT: "600000" - WITH_NVIDIA_PEERMEM: "0" - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - decode-context-parallel-size: 8 - dcp-comm-backend: a2a - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - gpu-memory-utilization: 0.92 - no-enable-flashinfer-autotune: true - max-model-len: 1048576 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' - attention-backend: TOKENSPEED_MLA - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - enable-prefix-caching: true - enable-prompt-tokens-details: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - max-num-seqs: 144 - max-num-batched-tokens: 8192 - prefix-match-unit: 128 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,7,18,34,53,64,75,100,128,256,512,1024,2048,4096,8192]}' - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c96-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c96-agentic.yaml deleted file mode 100644 index a76cc01a96..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c96-agentic.yaml +++ /dev/null @@ -1,128 +0,0 @@ -name: "kimik3-vllm-agg-gb200-tp8pp2-mooncake-c96-agentic" - -model: - path: "kimi-k3" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-728d3ad" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 4 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - MC_GID_INDEX: "3" - MC_STORE_MEMCPY: "1" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_SLICE_SIZE: "1048576" - MC_WORKERS_PER_CTX: "4" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_RPC_TIMEOUT: "600000" - WITH_NVIDIA_PEERMEM: "0" - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - decode-context-parallel-size: 8 - dcp-comm-backend: a2a - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - gpu-memory-utilization: 0.92 - no-enable-flashinfer-autotune: true - max-model-len: 1048576 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' - attention-backend: TOKENSPEED_MLA - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - enable-prefix-caching: true - enable-prompt-tokens-details: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - max-num-seqs: 192 - max-num-batched-tokens: 8192 - prefix-match-unit: 128 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,7,18,34,53,64,75,100,128,256,512,1024,2048,4096,8192]}' - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-mooncake-agentic.yaml deleted file mode 100644 index 09b9f3a7d8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-mooncake-agentic.yaml +++ /dev/null @@ -1,152 +0,0 @@ -name: "kimi-k3-vllm-agg-gb300-dcp8-dspark4-mooncake-agentic" - -model: - path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - frameworks: - dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -slurm: - time_limit: "04:00:00" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - -backend: - type: vllm - connector: null - mooncake_kv_store: - master_extra_args: - - "--default_kv_lease_ttl=60000" - - "--eviction_high_watermark_ratio=0.95" - - "--eviction_ratio=0.10" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "160GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - # Use the direct DCP a2a/gather kernels rather than the collective - # fallback, matching the B300 arm (kimik3_fp4_b300_vllm_mtp.sh). - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - ETCD_LEASE_TTL: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - PYTHONNOUSERSITE: "1" - TORCH_CUDA_ARCH_LIST: "10.0" - PYTHONHASHSEED: "42" - VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" - MC_SLICE_SIZE: "1048576" - MC_MAX_MR_SIZE: "4294967296" - WITH_NVIDIA_PEERMEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - decode-context-parallel-size: 8 - dcp-comm-backend: "a2a" - gpu-memory-utilization: 0.92 - max-num-batched-tokens: 16384 - trust-remote-code: true - language-model-only: true - load-format: "fastsafetensors" - moe-backend: "auto" - enable-flashinfer-autotune: true - enable-prefix-caching: true - prefix-match-unit: 128 - kv-cache-dtype: "fp8" - mamba-ssm-cache-dtype: "bfloat16" - stream-interval: 10 - attention-backend: "TOKENSPEED_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - max-cudagraph-capture-size: 1024 - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - comment: >- - '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' - -srun_options: - mem: "0" - container-remap-root: "" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - -benchmark: - type: custom - client_placement: head - concurrencies: [48, 52, 56] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - ENABLE_AGENTX_POWER: "1" - REQUIRE_POWER: "1" - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml deleted file mode 100644 index dce9b831cc..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml +++ /dev/null @@ -1,146 +0,0 @@ -name: "kimi-k3-vllm-agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic" - -model: - path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - frameworks: - dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -slurm: - time_limit: "04:00:00" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - -backend: - type: vllm - connector: null - mooncake_kv_store: - master_extra_args: - - "--default_kv_lease_ttl=60000" - - "--eviction_high_watermark_ratio=0.95" - - "--eviction_ratio=0.10" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "160GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - ETCD_LEASE_TTL: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - MC_SLICE_SIZE: "1048576" - MC_MAX_MR_SIZE: "4294967296" - WITH_NVIDIA_PEERMEM: "0" - VLLM_LOG_STATS_INTERVAL: "1" - - vllm_config: - aggregated: - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_offload":false}}' - served-model-name: "moonshotai/Kimi-K3" - load-format: "fastsafetensors" - kv-cache-dtype: "fp8" - mamba-ssm-cache-dtype: "bfloat16" - gpu-memory-utilization: 0.92 - tensor-parallel-size: 8 - decode-context-parallel-size: 8 - dcp-comm-backend: "a2a" - max-num-seqs: 2 - max-num-batched-tokens: 8192 - trust-remote-code: true - max-cudagraph-capture-size: 1024 - stream-interval: 10 - language-model-only: true - moe-backend: "auto" - enable-flashinfer-autotune: true - enable-cumem-allocator: true - attention-backend: "TOKENSPEED_MLA" - attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block"}' - enable-prefix-caching: true - prefix-match-unit: 128 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - comment: >- - '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' - -srun_options: - mem: "0" - container-remap-root: "" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 19401 - -benchmark: - type: custom - client_placement: head - concurrencies: [1, 4] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - ENABLE_AGENTX_POWER: "1" - REQUIRE_POWER: "1" - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp16dp2ep32-latency-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp16dp2ep32-latency-agentic.yaml deleted file mode 100644 index a4c9a03372..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp16dp2ep32-latency-agentic.yaml +++ /dev/null @@ -1,108 +0,0 @@ -name: "kimik3-vllm-agg-h200-tp16dp2ep32-latency-agentic" - -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - vllm: "0.1.dev19262+gb6bbf29dd.d20260727" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -resources: - gpu_type: "h200" - gpus_per_node: 8 - agg_nodes: 4 - agg_workers: 1 - gpus_per_agg: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - aggregated_environment: - CUDA_LAUNCH_BLOCKING: "1" - GLOO_SOCKET_IFNAME: "eth0" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - NCCL_SOCKET_IFNAME: "eth0" - NCCL_CUMEM_ENABLE: "1" - PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" - PYTHONNOUSERSITE: "1" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_ROUTED_DOWN_PROJ_STREAM_TOKEN_THRESHOLD: "0" - VLLM_USE_V2_MODEL_RUNNER: "1" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 - data-parallel-size: 2 - enable-expert-parallel: true - trust-remote-code: true - load-format: fastsafetensors - moe-backend: marlin - attention-backend: FLASHMLA - gpu-memory-utilization: 0.975 - max-num-seqs: 5 - max-num-batched-tokens: 4096 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' - enforce-eager: true - compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' - enable-prefix-caching: true - enable-prompt-tokens-details: true - no-enable-flashinfer-autotune: true - disable-custom-all-reduce: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - -srun_options: - container-remap-root: "" - -telemetry: - enabled: true - provider: dcgm-power - default_frequency: 1.0 - storage_subdir: power - required: true - startup_timeout_seconds: 120 - request_timeout_seconds: 2 - collector_join_timeout_seconds: 12 - dcgm_exporter: - container_image: dcgm-exporter - port: 9401 - -benchmark: - type: custom - concurrencies: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12] - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-balanced-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-balanced-agentic.yaml deleted file mode 100644 index ba2c7f226d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-balanced-agentic.yaml +++ /dev/null @@ -1,95 +0,0 @@ -name: "kimik3-vllm-agg-h200-tp8dp4ep32-balanced-agentic" - -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - vllm: "0.1.dev19262+gb6bbf29dd.d20260727" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -resources: - gpu_type: "h200" - gpus_per_node: 8 - agg_nodes: 4 - agg_workers: 1 - gpus_per_agg: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - aggregated_environment: - CUDA_LAUNCH_BLOCKING: "1" - GLOO_SOCKET_IFNAME: "eth0" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - NCCL_SOCKET_IFNAME: "eth0" - NCCL_CUMEM_ENABLE: "1" - PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" - PYTHONNOUSERSITE: "1" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_ROUTED_DOWN_PROJ_STREAM_TOKEN_THRESHOLD: "0" - VLLM_USE_V2_MODEL_RUNNER: "1" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - data-parallel-size: 4 - enable-expert-parallel: true - trust-remote-code: true - load-format: fastsafetensors - moe-backend: marlin - attention-backend: FLASHMLA - gpu-memory-utilization: 0.95 - max-num-seqs: 8 - max-num-batched-tokens: 4096 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' - enforce-eager: true - no-async-scheduling: true - compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' - enable-prefix-caching: true - enable-prompt-tokens-details: true - no-enable-flashinfer-autotune: true - disable-custom-all-reduce: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-vllm-simple-agentic.yaml deleted file mode 100644 index 8f76304fbb..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-vllm-simple-agentic.yaml +++ /dev/null @@ -1,96 +0,0 @@ -name: "kimik3-vllm-agg-h200-tp8dp4ep32-vllm-simple-agentic" - -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - vllm: "0.1.dev19262+gb6bbf29dd.d20260727" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -resources: - gpu_type: "h200" - gpus_per_node: 8 - agg_nodes: 4 - agg_workers: 1 - gpus_per_agg: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - aggregated_environment: - CUDA_LAUNCH_BLOCKING: "1" - GLOO_SOCKET_IFNAME: "eth0" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - NCCL_SOCKET_IFNAME: "eth0" - NCCL_CUMEM_ENABLE: "1" - PYTHONHASHSEED: "42" - PYTHONNOUSERSITE: "1" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_ROUTED_DOWN_PROJ_STREAM_TOKEN_THRESHOLD: "0" - VLLM_USE_V2_MODEL_RUNNER: "1" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - data-parallel-size: 4 - enable-expert-parallel: true - trust-remote-code: true - load-format: fastsafetensors - moe-backend: marlin - attention-backend: FLASHMLA - gpu-memory-utilization: 0.95 - max-num-seqs: 16 - max-num-batched-tokens: 4096 - speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":2,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' - enforce-eager: true - no-async-scheduling: true - compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' - enable-prefix-caching: true - enable-prompt-tokens-details: true - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":154250000000,"lazy_offload":false}}' - no-enable-flashinfer-autotune: true - disable-custom-all-reduce: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml deleted file mode 100644 index 5505fd5e2e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml +++ /dev/null @@ -1,176 +0,0 @@ -name: "kimi-k3-vllm-disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic" - -model: - path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - frameworks: - dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -slurm: - time_limit: "04:00:00" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: - prefill: true - mooncake_kv_store: - master_extra_args: - - "--default_kv_lease_ttl=60000" - - "--eviction_high_watermark_ratio=0.95" - - "--eviction_ratio=0.10" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "160GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - prefill_environment: &kimi_env - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" - DYN_REQUEST_PLANE: "tcp" - ETCD_LEASE_TTL: "600" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - MC_SLICE_SIZE: "1048576" - MC_MAX_MR_SIZE: "4294967296" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - WITH_NVIDIA_PEERMEM: "0" - NCCL_NET_PLUGIN: "none" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_TCP_AF_PRIO: "inet" - VLLM_SSM_CONV_STATE_LAYOUT: "DS" - decode_environment: - <<: *kimi_env - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "moonshotai/Kimi-K3" - prefix-match-unit: 128 - load-format: "fastsafetensors" - kv-cache-dtype: "fp8" - mamba-ssm-cache-dtype: "bfloat16" - gpu-memory-utilization: 0.92 - tensor-parallel-size: 8 - decode-context-parallel-size: 8 - cp-kv-cache-interleave-size: 1 - dcp-comm-backend: "a2a" - enable-flashinfer-autotune: true - enable-cumem-allocator: true - trust-remote-code: true - max-cudagraph-capture-size: 512 - stream-interval: 10 - language-model-only: true - attention-backend: "TOKENSPEED_MLA" - attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' - enable-prefix-caching: true - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "moonshotai/Kimi-K3" - prefix-match-unit: 128 - load-format: "fastsafetensors" - kv-cache-dtype: "fp8" - mamba-ssm-cache-dtype: "bfloat16" - gpu-memory-utilization: 0.92 - tensor-parallel-size: 8 - decode-context-parallel-size: 8 - cp-kv-cache-interleave-size: 1 - dcp-comm-backend: "a2a" - enable-flashinfer-autotune: true - enable-cumem-allocator: true - trust-remote-code: true - max-cudagraph-capture-size: 512 - stream-interval: 10 - language-model-only: true - attention-backend: "TOKENSPEED_MLA" - attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' - enable-prefix-caching: true - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - comment: >- - '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml deleted file mode 100644 index d904a31953..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml +++ /dev/null @@ -1,176 +0,0 @@ -name: "kimi-k3-vllm-disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic" - -model: - path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - frameworks: - dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -slurm: - time_limit: "04:00:00" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: - prefill: true - mooncake_kv_store: - master_extra_args: - - "--default_kv_lease_ttl=60000" - - "--eviction_high_watermark_ratio=0.95" - - "--eviction_ratio=0.10" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "160GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - prefill_environment: &kimi_env - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" - DYN_REQUEST_PLANE: "tcp" - ETCD_LEASE_TTL: "600" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - MC_SLICE_SIZE: "1048576" - MC_MAX_MR_SIZE: "4294967296" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - WITH_NVIDIA_PEERMEM: "0" - NCCL_NET_PLUGIN: "none" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_TCP_AF_PRIO: "inet" - VLLM_SSM_CONV_STATE_LAYOUT: "DS" - decode_environment: - <<: *kimi_env - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "moonshotai/Kimi-K3" - prefix-match-unit: 128 - load-format: "fastsafetensors" - kv-cache-dtype: "fp8" - mamba-ssm-cache-dtype: "bfloat16" - gpu-memory-utilization: 0.92 - tensor-parallel-size: 8 - decode-context-parallel-size: 8 - cp-kv-cache-interleave-size: 1 - dcp-comm-backend: "a2a" - enable-flashinfer-autotune: true - enable-cumem-allocator: true - trust-remote-code: true - max-cudagraph-capture-size: 512 - stream-interval: 10 - language-model-only: true - attention-backend: "TOKENSPEED_MLA" - attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' - enable-prefix-caching: true - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "moonshotai/Kimi-K3" - prefix-match-unit: 128 - load-format: "fastsafetensors" - kv-cache-dtype: "fp8" - mamba-ssm-cache-dtype: "bfloat16" - gpu-memory-utilization: 0.92 - tensor-parallel-size: 8 - decode-context-parallel-size: 8 - cp-kv-cache-interleave-size: 1 - dcp-comm-backend: "a2a" - enable-flashinfer-autotune: true - enable-cumem-allocator: true - trust-remote-code: true - max-cudagraph-capture-size: 512 - stream-interval: 10 - language-model-only: true - attention-backend: "TOKENSPEED_MLA" - attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' - enable-prefix-caching: true - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - comment: >- - '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml deleted file mode 100644 index 8661534b90..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml +++ /dev/null @@ -1,176 +0,0 @@ -name: "kimi-k3-vllm-disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic" - -model: - path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - frameworks: - dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -slurm: - time_limit: "04:00:00" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 6 - prefill_workers: 1 - decode_workers: 3 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: - prefill: true - mooncake_kv_store: - master_extra_args: - - "--default_kv_lease_ttl=60000" - - "--eviction_high_watermark_ratio=0.95" - - "--eviction_ratio=0.10" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "160GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - prefill_environment: &kimi_env - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" - DYN_REQUEST_PLANE: "tcp" - ETCD_LEASE_TTL: "600" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - MC_SLICE_SIZE: "1048576" - MC_MAX_MR_SIZE: "4294967296" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - WITH_NVIDIA_PEERMEM: "0" - NCCL_NET_PLUGIN: "none" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_TCP_AF_PRIO: "inet" - VLLM_SSM_CONV_STATE_LAYOUT: "DS" - decode_environment: - <<: *kimi_env - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "moonshotai/Kimi-K3" - prefix-match-unit: 128 - load-format: "fastsafetensors" - kv-cache-dtype: "fp8" - mamba-ssm-cache-dtype: "bfloat16" - gpu-memory-utilization: 0.92 - tensor-parallel-size: 8 - decode-context-parallel-size: 8 - cp-kv-cache-interleave-size: 1 - dcp-comm-backend: "a2a" - enable-flashinfer-autotune: true - enable-cumem-allocator: true - trust-remote-code: true - max-cudagraph-capture-size: 512 - stream-interval: 10 - language-model-only: true - attention-backend: "TOKENSPEED_MLA" - attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' - enable-prefix-caching: true - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "moonshotai/Kimi-K3" - prefix-match-unit: 128 - load-format: "fastsafetensors" - kv-cache-dtype: "fp8" - mamba-ssm-cache-dtype: "bfloat16" - gpu-memory-utilization: 0.92 - tensor-parallel-size: 8 - decode-context-parallel-size: 8 - cp-kv-cache-interleave-size: 1 - dcp-comm-backend: "a2a" - enable-flashinfer-autotune: true - enable-cumem-allocator: true - trust-remote-code: true - max-cudagraph-capture-size: 512 - stream-interval: 10 - language-model-only: true - attention-backend: "TOKENSPEED_MLA" - attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block"}' - enable-prefix-caching: true - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - comment: >- - '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml deleted file mode 100644 index 4c5a3e32c1..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml +++ /dev/null @@ -1,176 +0,0 @@ -name: "kimi-k3-vllm-disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic" - -model: - path: "moonshotai/Kimi-K3" - container: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:nightly-dc36fcce902a63eab06c1b93a5c4a5ee178a0c56" - frameworks: - dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -slurm: - time_limit: "04:00:00" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 6 - prefill_workers: 1 - decode_workers: 3 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: - prefill: true - mooncake_kv_store: - master_extra_args: - - "--default_kv_lease_ttl=60000" - - "--eviction_high_watermark_ratio=0.95" - - "--eviction_ratio=0.10" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "160GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - prefill_environment: &kimi_env - VLLM_USE_DIRECT_DCP_A2A: "1" - VLLM_USE_DIRECT_DCP_Q_GATHER: "1" - VLLM_USE_DIRECT_DCP_KV_GATHER: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: "0" - DYN_REQUEST_PLANE: "tcp" - ETCD_LEASE_TTL: "600" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" - MC_SLICE_SIZE: "1048576" - MC_MAX_MR_SIZE: "4294967296" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - WITH_NVIDIA_PEERMEM: "0" - NCCL_NET_PLUGIN: "none" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_TCP_AF_PRIO: "inet" - VLLM_SSM_CONV_STATE_LAYOUT: "DS" - decode_environment: - <<: *kimi_env - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "moonshotai/Kimi-K3" - prefix-match-unit: 128 - load-format: "fastsafetensors" - kv-cache-dtype: "fp8" - mamba-ssm-cache-dtype: "bfloat16" - gpu-memory-utilization: 0.92 - tensor-parallel-size: 8 - decode-context-parallel-size: 8 - cp-kv-cache-interleave-size: 1 - dcp-comm-backend: "a2a" - enable-flashinfer-autotune: true - enable-cumem-allocator: true - trust-remote-code: true - max-cudagraph-capture-size: 512 - stream-interval: 10 - language-model-only: true - attention-backend: "TOKENSPEED_MLA" - attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block"}' - enable-prefix-caching: true - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "moonshotai/Kimi-K3" - prefix-match-unit: 128 - load-format: "fastsafetensors" - kv-cache-dtype: "fp8" - mamba-ssm-cache-dtype: "bfloat16" - gpu-memory-utilization: 0.92 - tensor-parallel-size: 8 - decode-context-parallel-size: 8 - cp-kv-cache-interleave-size: 1 - dcp-comm-backend: "a2a" - enable-flashinfer-autotune: true - enable-cumem-allocator: true - trust-remote-code: true - max-cudagraph-capture-size: 512 - stream-interval: 10 - language-model-only: true - attention-backend: "TOKENSPEED_MLA" - attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"rejection_sample_method":"block"}' - enable-prefix-caching: true - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - comment: >- - '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p1d-dep4-dep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p1d-dep4-dep8-1k1k.yaml deleted file mode 100644 index 710c4d9796..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p1d-dep4-dep8-1k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep8-fp8-1k1k" - -# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 1D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 2 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1024x4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p1d-dep4-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p1d-dep4-tep8-1k1k.yaml deleted file mode 100644 index c1b9cf32d6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p1d-dep4-tep8-1k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-tep8-fp8-1k1k" - -# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 1D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 2 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8196 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "128x256" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p2d-dep4-dep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p2d-dep4-dep8-1k1k.yaml deleted file mode 100644 index 907633ba74..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p2d-dep4-dep8-1k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-dep8-fp8-1k1k" - -# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1024x4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p2d-dep4-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p2d-dep4-tep8-1k1k.yaml deleted file mode 100644 index 2e2bbc12a9..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p2d-dep4-tep8-1k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-1k1k" - -# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8196 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4x16x64" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p4d-dep4-tp4-marlin-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p4d-dep4-tp4-marlin-1k1k.yaml deleted file mode 100644 index 429aa015ea..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p4d-dep4-tp4-marlin-1k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p4d-dep4-tp4-marlin-fp8-1k1k" - -# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 4D TP4 decode (TP4, 4 GPU/worker = 1 node each). -# Decode uses Marlin MoE backend with expert-parallel disabled (1p4d only). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 4 - moe-backend: marlin - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8196 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "32" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p1d-dep4-dep16-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p1d-dep4-dep16-8k1k.yaml deleted file mode 100644 index c08d88b967..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p1d-dep4-dep16-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep16-fp8-8k1k" - -# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260710 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "512" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-tep4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-tep4-8k1k.yaml deleted file mode 100644 index 64e712e0c8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-tep4-8k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep4-fp8-8k1k" - -# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D TEP4 decode (TP4 EP, 4 GPU/worker = 1 node each). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 2 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260710 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 4 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 512 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-tep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-tep8-8k1k.yaml deleted file mode 100644 index 8606cf5c38..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-tep8-8k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-8k1k" - -# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260710 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4x16x32x64x128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/2p1d-dep4-dep16-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/2p1d-dep4-dep16-8k1k.yaml deleted file mode 100644 index 48a05fe68e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/2p1d-dep4-dep16-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep16-fp8-8k1k" - -# 2P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260710 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 4 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "512x1024" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/3p1d-dep4-dep16-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/3p1d-dep4-dep16-8k1k.yaml deleted file mode 100644 index b55718a647..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/3p1d-dep4-dep16-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-3p1d-dep4-dep16-fp8-8k1k" - -# 3P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 2 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260710 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 4 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/5p1d-dep4-dep16-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/5p1d-dep4-dep16-8k1k.yaml deleted file mode 100644 index c4dd774564..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/5p1d-dep4-dep16-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-5p1d-dep4-dep16-fp8-8k1k" - -# 5P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 3 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260710 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 5 - decode_nodes: 4 - prefill_workers: 5 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/1p1d-dep2-dep4-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/1p1d-dep2-dep4-1k1k.yaml deleted file mode 100644 index f57d7af090..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/1p1d-dep2-dep4-1k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-dep4-fp8-1k1k" - -# 1P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D DEP4 decode (TP1 DP4 EP, 4 GPU/worker = 1 node each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 1 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/1p2d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/1p2d-dep2-tep8-1k1k.yaml deleted file mode 100644 index b4f457654a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/1p2d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p2d-dep2-tep8-fp8-1k1k" - -# 1P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8196 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4x16x64x128x256" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p2d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p2d-dep2-tep8-1k1k.yaml deleted file mode 100644 index 6bba9ea864..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p2d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-2p2d-dep2-tep8-fp8-1k1k" - -# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 2 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8196 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "32" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p3d-dep2-dep4-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p3d-dep2-dep4-1k1k.yaml deleted file mode 100644 index de852e427a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p3d-dep2-dep4-1k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-2p3d-dep2-dep4-fp8-1k1k" - -# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 3D DEP4 decode (TP1 DP4 EP, 4 GPU/worker = 1 node each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 3 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 3 - prefill_workers: 2 - decode_workers: 3 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p4d-dep2-dep4-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p4d-dep2-dep4-1k1k.yaml deleted file mode 100644 index 8f7b7b140e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p4d-dep2-dep4-1k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-2p4d-dep2-dep4-fp8-1k1k" - -# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 4D DEP4 decode (TP1 DP4 EP, 4 GPU/worker = 1 node each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 2 - decode_workers: 4 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/4p2d-dep2-dep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/4p2d-dep2-dep8-1k1k.yaml deleted file mode 100644 index f6cf6a59f0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/4p2d-dep2-dep8-1k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-4p2d-dep2-dep8-fp8-1k1k" - -# 4P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 2 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 4 - prefill_workers: 4 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1024x4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p1d-dep2-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p1d-dep2-dep8-8k1k.yaml deleted file mode 100644 index ec72d22b08..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p1d-dep2-dep8-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-dep8-fp8-8k1k" - -# 1P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 2 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260710 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p1d-dep2-tep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p1d-dep2-tep8-8k1k.yaml deleted file mode 100644 index ebc16ae605..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p1d-dep2-tep8-8k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tep8-fp8-8k1k" - -# 1P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 2 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260710 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p2d-dep2-tep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p2d-dep2-tep8-8k1k.yaml deleted file mode 100644 index 65a6f797a6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p2d-dep2-tep8-8k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p2d-dep2-tep8-fp8-8k1k" - -# 1P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260710 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "32x64x128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p1d-dep2-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p1d-dep2-dep8-8k1k.yaml deleted file mode 100644 index fa51ba1792..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p1d-dep2-dep8-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-2p1d-dep2-dep8-fp8-8k1k" - -# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 2 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260710 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "512" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p2d-dep2-tep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p2d-dep2-tep8-8k1k.yaml deleted file mode 100644 index 79c7649689..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p2d-dep2-tep8-8k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-2p2d-dep2-tep8-fp8-8k1k" - -# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260710 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 2 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p4d-dep2-tep4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p4d-dep2-tep4-8k1k.yaml deleted file mode 100644 index fdb35c7fc6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p4d-dep2-tep4-8k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-2p4d-dep2-tep4-fp8-8k1k" - -# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 4D TEP4 decode (TP4 EP, 4 GPU/worker = 1 node each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260710 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 2 - decode_workers: 4 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 4 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 512 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/3p1d-dep2-dep16-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/3p1d-dep2-dep16-8k1k.yaml deleted file mode 100644 index f633da0857..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/3p1d-dep2-dep16-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-3p1d-dep2-dep16-fp8-8k1k" - -# 3P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 2 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260710 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 4 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "512" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/3p1d-dep2-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/3p1d-dep2-dep8-8k1k.yaml deleted file mode 100644 index 8335b2f7bf..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/3p1d-dep2-dep8-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-3p1d-dep2-dep8-fp8-8k1k" - -# 3P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 2 prefill + 2 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260710 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1024" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/6p1d-dep2-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/6p1d-dep2-dep8-8k1k.yaml deleted file mode 100644 index 94e7898237..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/6p1d-dep2-dep8-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-6p1d-dep2-dep8-fp8-8k1k" - -# 6P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 3 prefill + 2 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260710 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 2 - prefill_workers: 6 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-dep8-eagle3-c64-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-dep8-eagle3-c64-8k1k.yaml deleted file mode 100644 index 72fcd6cc2e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-dep8-eagle3-c64-8k1k.yaml +++ /dev/null @@ -1,116 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-dep8-mxfp8-8k1k-eagle3-c64" - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp8" - -identity: - model: - repo: "MiniMaxAI/MiniMax-M3-MXFP8" - revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - vllm: "0.26.1rc1.dev255+g5e35a6f4f" - -dynamo: - install: true - version: "1.4.0.dev20260730" - request_plane: "nats" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: "dynamo" - enable_multiple_frontends: false - -backend: - type: "vllm" - connector: null - - prefill_environment: &worker-environment - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: *worker-environment - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "64" - req_rate: "inf" - num_warmup_mult: 2 - random_range_ratio: 0.8 - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c1-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c1-8k1k.yaml deleted file mode 100644 index 990e540075..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c1-8k1k.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp4-mxfp8-8k1k-eagle3-c1" - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp8" - -identity: - model: - repo: "MiniMaxAI/MiniMax-M3-MXFP8" - revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - vllm: "0.26.1rc1.dev255+g5e35a6f4f" - -dynamo: - install: true - version: "1.4.0.dev20260730" - request_plane: "nats" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -frontend: - type: "dynamo" - enable_multiple_frontends: false - -backend: - type: "vllm" - connector: null - - prefill_environment: &worker-environment - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: *worker-environment - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 4 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1" - req_rate: "inf" - num_warmup_mult: 2 - random_range_ratio: 0.8 - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c8-8k1k.yaml deleted file mode 100644 index ff8f84cf47..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c8-8k1k.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp4-mxfp8-8k1k-eagle3-c8" - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp8" - -identity: - model: - repo: "MiniMaxAI/MiniMax-M3-MXFP8" - revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - vllm: "0.26.1rc1.dev255+g5e35a6f4f" - -dynamo: - install: true - version: "1.4.0.dev20260730" - request_plane: "nats" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -frontend: - type: "dynamo" - enable_multiple_frontends: false - -backend: - type: "vllm" - connector: null - - prefill_environment: &worker-environment - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: *worker-environment - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 4 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "8" - req_rate: "inf" - num_warmup_mult: 2 - random_range_ratio: 0.8 - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c1-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c1-8k1k.yaml deleted file mode 100644 index 4fa642e3d9..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c1-8k1k.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp8-mxfp8-8k1k-eagle3-c1" - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp8" - -identity: - model: - repo: "MiniMaxAI/MiniMax-M3-MXFP8" - revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - vllm: "0.26.1rc1.dev255+g5e35a6f4f" - -dynamo: - install: true - version: "1.4.0.dev20260730" - request_plane: "nats" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: "dynamo" - enable_multiple_frontends: false - -backend: - type: "vllm" - connector: null - - prefill_environment: &worker-environment - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: *worker-environment - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 8 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1" - req_rate: "inf" - num_warmup_mult: 2 - random_range_ratio: 0.8 - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c4-8k1k.yaml deleted file mode 100644 index 280819ae5a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c4-8k1k.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp8-mxfp8-8k1k-eagle3-c4" - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp8" - -identity: - model: - repo: "MiniMaxAI/MiniMax-M3-MXFP8" - revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - vllm: "0.26.1rc1.dev255+g5e35a6f4f" - -dynamo: - install: true - version: "1.4.0.dev20260730" - request_plane: "nats" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: "dynamo" - enable_multiple_frontends: false - -backend: - type: "vllm" - connector: null - - prefill_environment: &worker-environment - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: *worker-environment - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 8 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4" - req_rate: "inf" - num_warmup_mult: 2 - random_range_ratio: 0.8 - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c8-8k1k.yaml deleted file mode 100644 index 94322297f0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c8-8k1k.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp8-mxfp8-8k1k-eagle3-c8" - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp8" - -identity: - model: - repo: "MiniMaxAI/MiniMax-M3-MXFP8" - revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - vllm: "0.26.1rc1.dev255+g5e35a6f4f" - -dynamo: - install: true - version: "1.4.0.dev20260730" - request_plane: "nats" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: "dynamo" - enable_multiple_frontends: false - -backend: - type: "vllm" - connector: null - - prefill_environment: &worker-environment - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: *worker-environment - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 8 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "8" - req_rate: "inf" - num_warmup_mult: 2 - random_range_ratio: 0.8 - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/2p1d-dep2-dep8-eagle3-c512-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/2p1d-dep2-dep8-eagle3-c512-8k1k.yaml deleted file mode 100644 index e75a8eb06a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/2p1d-dep2-dep8-eagle3-c512-8k1k.yaml +++ /dev/null @@ -1,116 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-2p1d-dep2-dep8-mxfp8-8k1k-eagle3-c512" - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp8" - -identity: - model: - repo: "MiniMaxAI/MiniMax-M3-MXFP8" - revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - vllm: "0.26.1rc1.dev255+g5e35a6f4f" - -dynamo: - install: true - version: "1.4.0.dev20260730" - request_plane: "nats" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: "dynamo" - enable_multiple_frontends: false - -backend: - type: "vllm" - connector: null - - prefill_environment: &worker-environment - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: *worker-environment - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "512" - req_rate: "inf" - num_warmup_mult: 2 - random_range_ratio: 0.8 - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/3p1d-dep2-dep8-eagle3-c256-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/3p1d-dep2-dep8-eagle3-c256-8k1k.yaml deleted file mode 100644 index 3a7753467c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/3p1d-dep2-dep8-eagle3-c256-8k1k.yaml +++ /dev/null @@ -1,116 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-3p1d-dep2-dep8-mxfp8-8k1k-eagle3-c256" - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp8" - -identity: - model: - repo: "MiniMaxAI/MiniMax-M3-MXFP8" - revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - vllm: "0.26.1rc1.dev255+g5e35a6f4f" - -dynamo: - install: true - version: "1.4.0.dev20260730" - request_plane: "nats" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: "dynamo" - enable_multiple_frontends: false - -backend: - type: "vllm" - connector: null - - prefill_environment: &worker-environment - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: *worker-environment - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256" - req_rate: "inf" - num_warmup_mult: 2 - random_range_ratio: 0.8 - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c1024-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c1024-8k1k.yaml deleted file mode 100644 index 4b25736627..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c1024-8k1k.yaml +++ /dev/null @@ -1,116 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-4p1d-dep2-dep8-mxfp8-8k1k-eagle3-c1024" - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp8" - -identity: - model: - repo: "MiniMaxAI/MiniMax-M3-MXFP8" - revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - vllm: "0.26.1rc1.dev255+g5e35a6f4f" - -dynamo: - install: true - version: "1.4.0.dev20260730" - request_plane: "nats" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 4 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: "dynamo" - enable_multiple_frontends: false - -backend: - type: "vllm" - connector: null - - prefill_environment: &worker-environment - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: *worker-environment - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1024" - req_rate: "inf" - num_warmup_mult: 2 - random_range_ratio: 0.8 - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c2048-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c2048-8k1k.yaml deleted file mode 100644 index c9537a9fcd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c2048-8k1k.yaml +++ /dev/null @@ -1,116 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-4p1d-dep2-dep8-mxfp8-8k1k-eagle3-c2048" - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp8" - -identity: - model: - repo: "MiniMaxAI/MiniMax-M3-MXFP8" - revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - vllm: "0.26.1rc1.dev255+g5e35a6f4f" - -dynamo: - install: true - version: "1.4.0.dev20260730" - request_plane: "nats" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 4 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: "dynamo" - enable_multiple_frontends: false - -backend: - type: "vllm" - connector: null - - prefill_environment: &worker-environment - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: *worker-environment - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" - num_warmup_mult: 2 - random_range_ratio: 0.8 - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/6p1d-dep2-dep8-eagle3-c2048-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/6p1d-dep2-dep8-eagle3-c2048-8k1k.yaml deleted file mode 100644 index 601864149f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/6p1d-dep2-dep8-eagle3-c2048-8k1k.yaml +++ /dev/null @@ -1,116 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-6p1d-dep2-dep8-mxfp8-8k1k-eagle3-c2048" - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp8" - -identity: - model: - repo: "MiniMaxAI/MiniMax-M3-MXFP8" - revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - vllm: "0.26.1rc1.dev255+g5e35a6f4f" - -dynamo: - install: true - version: "1.4.0.dev20260730" - request_plane: "nats" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 2 - prefill_workers: 6 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: "dynamo" - enable_multiple_frontends: false - -backend: - type: "vllm" - connector: null - - prefill_environment: &worker-environment - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: *worker-environment - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: "fp8" - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" - num_warmup_mult: 2 - random_range_ratio: 0.8 - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml deleted file mode 100644 index 421a096254..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml +++ /dev/null @@ -1,163 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - version: "1.4.0.dev20260730" - install: true - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - env: - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true - prefill_environment: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - decode_environment: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml deleted file mode 100644 index 3bed52725e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml +++ /dev/null @@ -1,163 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p1d-tep4-tp4-c1-fp4-eval-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - version: "1.4.0.dev20260730" - install: true - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - env: - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true - prefill_environment: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - decode_environment: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml deleted file mode 100644 index fb4a6bf759..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml +++ /dev/null @@ -1,163 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p1d-tp2-tp4-c20-c24-fp4-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - version: "1.4.0.dev20260730" - install: true - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - env: - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true - prefill_environment: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - decode_environment: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml deleted file mode 100644 index 755d2a8a12..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml +++ /dev/null @@ -1,163 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p1d-tp2-tp4-c20-c24-fp4-eval-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - version: "1.4.0.dev20260730" - install: true - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - env: - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true - prefill_environment: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - decode_environment: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml deleted file mode 100644 index 7bf7fbb74e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml +++ /dev/null @@ -1,166 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p3d-dep4-tp4-c24-fp4-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - version: "1.4.0.dev20260730" - install: true - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 1 - decode_nodes: 3 - prefill_workers: 1 - decode_workers: 3 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - env: - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true - prefill_environment: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - decode_environment: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml deleted file mode 100644 index 261f1fe6d2..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml +++ /dev/null @@ -1,166 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p3d-dep4-tp4-c24-fp4-eval-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - version: "1.4.0.dev20260730" - install: true - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 1 - decode_nodes: 3 - prefill_workers: 1 - decode_workers: 3 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - env: - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 4 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true - prefill_environment: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - decode_environment: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml deleted file mode 100644 index 8193a1f919..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml +++ /dev/null @@ -1,163 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p3d-tp2-tp2-c48-fp4-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - version: "1.4.0.dev20260730" - install: true - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: true - prefill_nodes: 1 - decode_nodes: 3 - prefill_workers: 1 - decode_workers: 3 - gpus_per_prefill: 2 - gpus_per_decode: 2 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - env: - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true - prefill_environment: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - decode_environment: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml deleted file mode 100644 index 4a174c2c28..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml +++ /dev/null @@ -1,163 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p3d-tp2-tp2-c48-fp4-eval-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - version: "1.4.0.dev20260730" - install: true - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: true - prefill_nodes: 1 - decode_nodes: 3 - prefill_workers: 1 - decode_workers: 3 - gpus_per_prefill: 2 - gpus_per_decode: 2 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - env: - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true - prefill_environment: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - decode_environment: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml deleted file mode 100644 index fe2d56097c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml +++ /dev/null @@ -1,163 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-2p5d-tp2-tp2-c120-fp4-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - version: "1.4.0.dev20260730" - install: true - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: true - prefill_nodes: 2 - decode_nodes: 5 - prefill_workers: 2 - decode_workers: 5 - gpus_per_prefill: 2 - gpus_per_decode: 2 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - env: - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true - prefill_environment: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - decode_environment: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml deleted file mode 100644 index bf01406d14..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml +++ /dev/null @@ -1,163 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-2p5d-tp2-tp2-c120-fp4-eval-agentic" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" - -identity: - model: - repo: "nvidia/MiniMax-M3-NVFP4" - container: - image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - frameworks: - dynamo: "1.4.0.dev20260730" - -dynamo: - version: "1.4.0.dev20260730" - install: true - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: true - prefill_nodes: 2 - decode_nodes: 5 - prefill_workers: 2 - decode_workers: 5 - gpus_per_prefill: 2 - gpus_per_decode: 2 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -environment: - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - env: - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_TCP_CONNECT_TIMEOUT: "120" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - env: - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "200GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":true}}]}}' - enable-cumem-allocator: true - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - kv-cache-dtype: "fp8" - block-size: 128 - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - max-num-seqs: 1024 - stream-interval: 20 - gpu-memory-utilization: 0.9 - tensor-parallel-size: 2 - enable-expert-parallel: false - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"enforce_handshake_compat":false,"read_validation_timeout":30.0}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - enable-cumem-allocator: true - prefill_environment: &worker_environment - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - UCX_MEMTYPE_CACHE: "n" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - decode_environment: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p1d-dep2-tp4-c1-c16-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p1d-dep2-tp4-c1-c16-8k1k.yaml deleted file mode 100644 index ebfbfc11e0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p1d-dep2-tp4-c1-c16-8k1k.yaml +++ /dev/null @@ -1,62 +0,0 @@ -name: "minimax-m3-vllm-disagg-b200-1p1d-fp4-dep2-tp4-8k1k" -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" -resources: - gpu_type: "b200" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 4 -dynamo: {install: true, version: 1.3.0.dev20260710} -frontend: {type: dynamo, enable_multiple_frontends: false} -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - prefill_environment: {VLLM_FLOAT32_MATMUL_PRECISION: high, UCX_TCP_AF_PRIO: "inet"} - decode_environment: {VLLM_FLOAT32_MATMUL_PRECISION: high, UCX_TCP_AF_PRIO: "inet"} - vllm_config: - prefill: - served-model-name: nvidia/MiniMax-M3-NVFP4 - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - decode: - served-model-name: nvidia/MiniMax-M3-NVFP4 - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 -health_check: {max_attempts: 360, interval_seconds: 10} -benchmark: {type: "sa-bench", isl: 8192, osl: 1024, random_range_ratio: 0.8, concurrencies: "1x4x8x16", req_rate: "inf"} diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p2d-dep2-tp4-c64-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p2d-dep2-tp4-c64-8k1k.yaml deleted file mode 100644 index b76ccf264a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p2d-dep2-tp4-c64-8k1k.yaml +++ /dev/null @@ -1,125 +0,0 @@ -name: "minimax-m3-vllm-disagg-b200-1p2d-fp4-dep2-tp4-c64-8k1k" -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" -resources: - gpu_type: "b200" - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 2 - decode_nodes: 2 - decode_workers: 2 - spread_workers: true - gpus_per_decode: 4 - gpus_per_node: 8 -dynamo: {install: true, version: 1.3.0.dev20260710} -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: vllm - reasoning-parser: minimax_m3 -backend: - type: vllm - connector: null - prefill_environment: - VLLM_USE_DEEP_GEMM: '1' - VLLM_SKIP_P2P_CHECK: '1' - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' - NVIDIA_GDRCOPY: '1' - PYTHONUNBUFFERED: '1' - VLLM_LOG_STATS_INTERVAL: '1' - NVSHMEM_IB_ENABLE_IBGDA: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NCCL_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' - VLLM_USE_NCCL_SYMM_MEM: '1' - VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' - VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TCP_AF_PRIO: "inet" - decode_environment: - VLLM_USE_DEEP_GEMM: '1' - VLLM_SKIP_P2P_CHECK: '1' - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' - NVIDIA_GDRCOPY: '1' - PYTHONUNBUFFERED: '1' - VLLM_LOG_STATS_INTERVAL: '1' - NVSHMEM_IB_ENABLE_IBGDA: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NCCL_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' - VLLM_USE_NCCL_SYMM_MEM: '1' - VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' - VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' - VLLM_MOE_DP_CHUNK_SIZE: '384' - VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: '8192' - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TCP_AF_PRIO: "inet" - vllm_config: - prefill: - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - enable-expert-parallel: true - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - max-model-len: 9472 - max-num-seqs: 16 - enforce-eager: true - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - no-enable-chunked-prefill: true - kv-cache-dtype: fp8 - async-scheduling: true - no-enable-prefix-caching: true - trust-remote-code: true - served-model-name: nvidia/MiniMax-M3-NVFP4 - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - decode: - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: false - max-model-len: 9472 - max-num-seqs: 32 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+rms_norm"],"pass_config":{}}' - kv-cache-dtype: fp8 - all2all-backend: deepep_low_latency - async-scheduling: true - stream-interval: 32 - enable-dbo: true - dbo-decode-token-threshold: 32 - no-enable-prefix-caching: true - trust-remote-code: true - served-model-name: nvidia/MiniMax-M3-NVFP4 - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - block-size: 128 - language-model-only: true -health_check: {max_attempts: 360, interval_seconds: 10} -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "64" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/2p2d-dep2-tp4-c128-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/2p2d-dep2-tp4-c128-8k1k.yaml deleted file mode 100644 index e09134da34..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/2p2d-dep2-tp4-c128-8k1k.yaml +++ /dev/null @@ -1,125 +0,0 @@ -name: "minimax-m3-vllm-disagg-b200-2p2d-fp4-dep2-tp4-c128-8k1k" -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" -resources: - gpu_type: "b200" - prefill_nodes: 2 - prefill_workers: 2 - gpus_per_prefill: 2 - decode_nodes: 2 - decode_workers: 2 - spread_workers: true - gpus_per_decode: 4 - gpus_per_node: 8 -dynamo: {install: true, version: 1.3.0.dev20260710} -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: vllm - reasoning-parser: minimax_m3 -backend: - type: vllm - connector: null - prefill_environment: - VLLM_USE_DEEP_GEMM: '1' - VLLM_SKIP_P2P_CHECK: '1' - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' - NVIDIA_GDRCOPY: '1' - PYTHONUNBUFFERED: '1' - VLLM_LOG_STATS_INTERVAL: '1' - NVSHMEM_IB_ENABLE_IBGDA: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NCCL_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' - VLLM_USE_NCCL_SYMM_MEM: '1' - VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' - VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TCP_AF_PRIO: "inet" - decode_environment: - VLLM_USE_DEEP_GEMM: '1' - VLLM_SKIP_P2P_CHECK: '1' - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' - NVIDIA_GDRCOPY: '1' - PYTHONUNBUFFERED: '1' - VLLM_LOG_STATS_INTERVAL: '1' - NVSHMEM_IB_ENABLE_IBGDA: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NCCL_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' - VLLM_USE_NCCL_SYMM_MEM: '1' - VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' - VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' - VLLM_MOE_DP_CHUNK_SIZE: '384' - VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: '8192' - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TCP_AF_PRIO: "inet" - vllm_config: - prefill: - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - enable-expert-parallel: true - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - max-model-len: 9472 - max-num-seqs: 16 - enforce-eager: true - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - no-enable-chunked-prefill: true - kv-cache-dtype: fp8 - async-scheduling: true - no-enable-prefix-caching: true - trust-remote-code: true - served-model-name: nvidia/MiniMax-M3-NVFP4 - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - decode: - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: false - max-model-len: 9472 - max-num-seqs: 64 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+rms_norm"],"pass_config":{}}' - kv-cache-dtype: fp8 - all2all-backend: deepep_low_latency - async-scheduling: true - stream-interval: 32 - enable-dbo: true - dbo-decode-token-threshold: 32 - no-enable-prefix-caching: true - trust-remote-code: true - served-model-name: nvidia/MiniMax-M3-NVFP4 - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - block-size: 128 - language-model-only: true -health_check: {max_attempts: 360, interval_seconds: 10} -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c256-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c256-8k1k.yaml deleted file mode 100644 index d07384822f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c256-8k1k.yaml +++ /dev/null @@ -1,125 +0,0 @@ -name: "minimax-m3-vllm-disagg-b200-3p2d-fp4-dep2-tp4-c256-8k1k" -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" -resources: - gpu_type: "b200" - prefill_nodes: 3 - prefill_workers: 3 - gpus_per_prefill: 2 - decode_nodes: 2 - decode_workers: 2 - spread_workers: true - gpus_per_decode: 4 - gpus_per_node: 8 -dynamo: {install: true, version: 1.3.0.dev20260710} -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: vllm - reasoning-parser: minimax_m3 -backend: - type: vllm - connector: null - prefill_environment: - VLLM_USE_DEEP_GEMM: '1' - VLLM_SKIP_P2P_CHECK: '1' - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' - NVIDIA_GDRCOPY: '1' - PYTHONUNBUFFERED: '1' - VLLM_LOG_STATS_INTERVAL: '1' - NVSHMEM_IB_ENABLE_IBGDA: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NCCL_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' - VLLM_USE_NCCL_SYMM_MEM: '1' - VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' - VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TCP_AF_PRIO: "inet" - decode_environment: - VLLM_USE_DEEP_GEMM: '1' - VLLM_SKIP_P2P_CHECK: '1' - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' - NVIDIA_GDRCOPY: '1' - PYTHONUNBUFFERED: '1' - VLLM_LOG_STATS_INTERVAL: '1' - NVSHMEM_IB_ENABLE_IBGDA: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NCCL_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' - VLLM_USE_NCCL_SYMM_MEM: '1' - VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' - VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' - VLLM_MOE_DP_CHUNK_SIZE: '384' - VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: '8192' - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TCP_AF_PRIO: "inet" - vllm_config: - prefill: - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - enable-expert-parallel: true - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - max-model-len: 9472 - max-num-seqs: 16 - enforce-eager: true - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - no-enable-chunked-prefill: true - kv-cache-dtype: fp8 - async-scheduling: true - no-enable-prefix-caching: true - trust-remote-code: true - served-model-name: nvidia/MiniMax-M3-NVFP4 - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - decode: - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: false - max-model-len: 9472 - max-num-seqs: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+rms_norm"],"pass_config":{}}' - kv-cache-dtype: fp8 - all2all-backend: deepep_low_latency - async-scheduling: true - stream-interval: 32 - enable-dbo: true - dbo-decode-token-threshold: 32 - no-enable-prefix-caching: true - trust-remote-code: true - served-model-name: nvidia/MiniMax-M3-NVFP4 - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - block-size: 128 - language-model-only: true -health_check: {max_attempts: 360, interval_seconds: 10} -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "256" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c512-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c512-8k1k.yaml deleted file mode 100644 index 73e64c6828..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c512-8k1k.yaml +++ /dev/null @@ -1,125 +0,0 @@ -name: "minimax-m3-vllm-disagg-b200-3p2d-fp4-dep2-tp4-c512-8k1k" -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" -resources: - gpu_type: "b200" - prefill_nodes: 3 - prefill_workers: 3 - gpus_per_prefill: 2 - decode_nodes: 2 - decode_workers: 2 - spread_workers: true - gpus_per_decode: 4 - gpus_per_node: 8 -dynamo: {install: true, version: 1.3.0.dev20260710} -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: vllm - reasoning-parser: minimax_m3 -backend: - type: vllm - connector: null - prefill_environment: - VLLM_USE_DEEP_GEMM: '1' - VLLM_SKIP_P2P_CHECK: '1' - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' - NVIDIA_GDRCOPY: '1' - PYTHONUNBUFFERED: '1' - VLLM_LOG_STATS_INTERVAL: '1' - NVSHMEM_IB_ENABLE_IBGDA: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NCCL_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' - VLLM_USE_NCCL_SYMM_MEM: '1' - VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' - VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TCP_AF_PRIO: "inet" - decode_environment: - VLLM_USE_DEEP_GEMM: '1' - VLLM_SKIP_P2P_CHECK: '1' - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' - NVIDIA_GDRCOPY: '1' - PYTHONUNBUFFERED: '1' - VLLM_LOG_STATS_INTERVAL: '1' - NVSHMEM_IB_ENABLE_IBGDA: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NCCL_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' - VLLM_USE_NCCL_SYMM_MEM: '1' - VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' - VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' - VLLM_MOE_DP_CHUNK_SIZE: '384' - VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: '8192' - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TCP_AF_PRIO: "inet" - vllm_config: - prefill: - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - enable-expert-parallel: true - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - max-model-len: 9472 - max-num-seqs: 16 - enforce-eager: true - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - no-enable-chunked-prefill: true - kv-cache-dtype: fp8 - async-scheduling: true - no-enable-prefix-caching: true - trust-remote-code: true - served-model-name: nvidia/MiniMax-M3-NVFP4 - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - decode: - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: false - max-model-len: 9472 - max-num-seqs: 256 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+rms_norm"],"pass_config":{}}' - kv-cache-dtype: fp8 - all2all-backend: deepep_low_latency - async-scheduling: true - stream-interval: 32 - enable-dbo: true - dbo-decode-token-threshold: 32 - no-enable-prefix-caching: true - trust-remote-code: true - served-model-name: nvidia/MiniMax-M3-NVFP4 - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - block-size: 128 - language-model-only: true -health_check: {max_attempts: 360, interval_seconds: 10} -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "512" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/4p2d-dep2-tp4-c1024-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/4p2d-dep2-tp4-c1024-8k1k.yaml deleted file mode 100644 index 24b0af93bd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/4p2d-dep2-tp4-c1024-8k1k.yaml +++ /dev/null @@ -1,125 +0,0 @@ -name: "minimax-m3-vllm-disagg-b200-4p2d-fp4-dep2-tp4-c1024-8k1k" -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" -resources: - gpu_type: "b200" - prefill_nodes: 4 - prefill_workers: 4 - gpus_per_prefill: 2 - decode_nodes: 2 - decode_workers: 2 - spread_workers: true - gpus_per_decode: 4 - gpus_per_node: 8 -dynamo: {install: true, version: 1.3.0.dev20260710} -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: vllm - reasoning-parser: minimax_m3 -backend: - type: vllm - connector: null - prefill_environment: - VLLM_USE_DEEP_GEMM: '1' - VLLM_SKIP_P2P_CHECK: '1' - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' - NVIDIA_GDRCOPY: '1' - PYTHONUNBUFFERED: '1' - VLLM_LOG_STATS_INTERVAL: '1' - NVSHMEM_IB_ENABLE_IBGDA: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NCCL_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' - VLLM_USE_NCCL_SYMM_MEM: '1' - VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' - VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TCP_AF_PRIO: "inet" - decode_environment: - VLLM_USE_DEEP_GEMM: '1' - VLLM_SKIP_P2P_CHECK: '1' - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' - NVIDIA_GDRCOPY: '1' - PYTHONUNBUFFERED: '1' - VLLM_LOG_STATS_INTERVAL: '1' - NVSHMEM_IB_ENABLE_IBGDA: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - NCCL_TIMEOUT: '1800' - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' - VLLM_USE_FLASHINFER_MOE_FP4: '1' - VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' - VLLM_USE_NCCL_SYMM_MEM: '1' - VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' - VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' - VLLM_MOE_DP_CHUNK_SIZE: '384' - VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: '8192' - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TCP_AF_PRIO: "inet" - vllm_config: - prefill: - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - enable-expert-parallel: true - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - max-model-len: 9472 - max-num-seqs: 16 - enforce-eager: true - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - no-enable-chunked-prefill: true - kv-cache-dtype: fp8 - async-scheduling: true - no-enable-prefix-caching: true - trust-remote-code: true - served-model-name: nvidia/MiniMax-M3-NVFP4 - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - decode: - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: false - max-model-len: 9472 - max-num-seqs: 512 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+rms_norm"],"pass_config":{}}' - kv-cache-dtype: fp8 - all2all-backend: deepep_low_latency - async-scheduling: true - stream-interval: 32 - enable-dbo: true - dbo-decode-token-threshold: 32 - no-enable-prefix-caching: true - trust-remote-code: true - served-model-name: nvidia/MiniMax-M3-NVFP4 - no-enable-flashinfer-autotune: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - block-size: 128 - language-model-only: true -health_check: {max_attempts: 360, interval_seconds: 10} -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1024" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p1d-tp4-dep4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p1d-tp4-dep4-eagle3-8k1k.yaml deleted file mode 100644 index a655ade331..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p1d-tp4-dep4-eagle3-8k1k.yaml +++ /dev/null @@ -1,98 +0,0 @@ -name: "minimax-m3-vllm-disagg-b200-1p1d-tp4-dep4-fp4-8k1k-eagle3" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: fp4 - -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 0 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - allow_prefill_decode_colocation_across_nodes: true - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm - UCX_TLS: cuda_copy,cuda_ipc,rc - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm - VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: cutlass - UCX_TLS: cuda_copy,cuda_ipc,rc - VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: cutlass - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - req_rate: inf - num_warmup_mult: 0 - random_range_ratio: 0.8 - use_chat_template: true - concurrencies: "256x512" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p1d-tp4-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p1d-tp4-tp4-eagle3-8k1k.yaml deleted file mode 100644 index ff498c89e7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p1d-tp4-tp4-eagle3-8k1k.yaml +++ /dev/null @@ -1,91 +0,0 @@ -name: "minimax-m3-vllm-disagg-b200-1p1d-tp4-tp4-fp4-8k1k-eagle3" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: fp4 - -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 0 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - allow_prefill_decode_colocation_across_nodes: true - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: cutlass - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - req_rate: inf - num_warmup_mult: 0 - random_range_ratio: 0.8 - use_chat_template: true - concurrencies: "192x256x512" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p2d-tp4-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p2d-tp4-tp4-eagle3-8k1k.yaml deleted file mode 100644 index a7909a5fbe..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p2d-tp4-tp4-eagle3-8k1k.yaml +++ /dev/null @@ -1,91 +0,0 @@ -name: "minimax-m3-vllm-disagg-b200-1p2d-tp4-tp4-fp4-8k1k-eagle3" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: fp4 - -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - allow_prefill_decode_colocation_across_nodes: true - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: cutlass - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - req_rate: inf - num_warmup_mult: 0 - random_range_ratio: 0.8 - use_chat_template: true - concurrencies: "32x64x96" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p3d-tp4-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p3d-tp4-tp4-eagle3-8k1k.yaml deleted file mode 100644 index 125b2ed529..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p3d-tp4-tp4-eagle3-8k1k.yaml +++ /dev/null @@ -1,91 +0,0 @@ -name: "minimax-m3-vllm-disagg-b200-1p3d-tp4-tp4-fp4-8k1k-eagle3" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: fp4 - -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 3 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - allow_prefill_decode_colocation_across_nodes: true - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: cutlass - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - req_rate: inf - num_warmup_mult: 0 - random_range_ratio: 0.8 - use_chat_template: true - concurrencies: "192" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p4d-tp4-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p4d-tp4-tp4-eagle3-8k1k.yaml deleted file mode 100644 index 3c867e3765..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p4d-tp4-tp4-eagle3-8k1k.yaml +++ /dev/null @@ -1,89 +0,0 @@ -name: "minimax-m3-vllm-disagg-b200-1p4d-tp4-tp4-fp4-8k1k-eagle3" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: fp4 - -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - allow_prefill_decode_colocation_across_nodes: true - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - req_rate: inf - num_warmup_mult: 0 - random_range_ratio: 0.8 - use_chat_template: true - concurrencies: "4x8x16x32x64" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p6d-tp4-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p6d-tp4-tp4-eagle3-8k1k.yaml deleted file mode 100644 index 792df6db7d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p6d-tp4-tp4-eagle3-8k1k.yaml +++ /dev/null @@ -1,91 +0,0 @@ -name: "minimax-m3-vllm-disagg-b200-1p6d-tp4-tp4-fp4-8k1k-eagle3" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: fp4 - -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 3 - prefill_workers: 1 - decode_workers: 6 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - allow_prefill_decode_colocation_across_nodes: true - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: cutlass - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - req_rate: inf - num_warmup_mult: 0 - random_range_ratio: 0.8 - use_chat_template: true - concurrencies: "12x24x48" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p1d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p1d-dep2-tep8-1k1k.yaml deleted file mode 100644 index 486af05573..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p1d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p1d-fp4-dep2-tep8-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4x16x64x128x4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p1d-dep2-tp4-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p1d-dep2-tp4-1k1k.yaml deleted file mode 100644 index 532b78a103..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p1d-dep2-tp4-1k1k.yaml +++ /dev/null @@ -1,82 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p1d-fp4-dep2-tp4-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - - prefill_environment: - UCX_TLS: "cuda_ipc,cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_ipc,cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1x4x8x16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p2d-dep2-dep4-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p2d-dep2-dep4-1k1k.yaml deleted file mode 100644 index fde8442a18..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p2d-dep2-dep4-1k1k.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p2d-fp4-dep2-dep4-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p1d-dep2-dep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p1d-dep2-dep8-1k1k.yaml deleted file mode 100644 index ed3b5f9950..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p1d-dep2-dep8-1k1k.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p1d-fp4-dep2-dep8-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "512x4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p1d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p1d-dep2-tep8-1k1k.yaml deleted file mode 100644 index 0784283b91..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p1d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p1d-fp4-dep2-tep8-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "32" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p2d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p2d-dep2-tep8-1k1k.yaml deleted file mode 100644 index 59c52da00c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p2d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p2d-fp4-dep2-tep8-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/3p2d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/3p2d-dep2-tep8-1k1k.yaml deleted file mode 100644 index 7e9f7dec31..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/3p2d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-3p2d-fp4-dep2-tep8-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 3 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p1d-tp1-tp4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p1d-tp1-tp4-8k1k.yaml deleted file mode 100644 index 8ca39d776b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p1d-tp1-tp4-8k1k.yaml +++ /dev/null @@ -1,73 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p1d-tp1-tp4-fp4-8k1k" -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 0 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 1 - gpus_per_decode: 4 -dynamo: - install: true - version: 1.3.0.dev20260713 -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - enforce-eager: true - max-num-batched-tokens: 8192 - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 512 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - allow_prefill_decode_colocation: true -health_check: - max_attempts: 360 - interval_seconds: 10 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "8x16x24x32x48x64" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p1d-tp1-tp8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p1d-tp1-tp8-8k1k.yaml deleted file mode 100644 index f85ca98cf1..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p1d-tp1-tp8-8k1k.yaml +++ /dev/null @@ -1,72 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p1d-tp1-tp8-fp4-8k1k" -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 1 - gpus_per_decode: 8 -dynamo: - install: true - version: 1.3.0.dev20260713 -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - enforce-eager: true - max-num-batched-tokens: 8192 - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 8 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 256 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 256 -health_check: - max_attempts: 360 - interval_seconds: 10 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1x2x4x8x16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/2p1d-tp1-tp4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/2p1d-tp1-tp4-8k1k.yaml deleted file mode 100644 index 71ed9f4c92..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/2p1d-tp1-tp4-8k1k.yaml +++ /dev/null @@ -1,73 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p1d-tp1-tp4-fp4-8k1k" -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 0 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 1 - gpus_per_decode: 4 -dynamo: - install: true - version: 1.3.0.dev20260713 -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - enforce-eager: true - max-num-batched-tokens: 8192 - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 512 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - allow_prefill_decode_colocation: true -health_check: - max_attempts: 360 - interval_seconds: 10 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/3p1d-tp1-tp4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/3p1d-tp1-tp4-8k1k.yaml deleted file mode 100644 index 476d21b310..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/3p1d-tp1-tp4-8k1k.yaml +++ /dev/null @@ -1,73 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-3p1d-tp1-tp4-fp4-8k1k" -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 0 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 1 - gpus_per_decode: 4 -dynamo: - install: true - version: 1.3.0.dev20260713 -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: - allow_prefill_decode_colocation: true - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - enforce-eager: true - max-num-batched-tokens: 8192 - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 512 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 -health_check: - max_attempts: 360 - interval_seconds: 10 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/4p1d-tp1-tep4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/4p1d-tp1-tep4-8k1k.yaml deleted file mode 100644 index f01976cc6f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/4p1d-tp1-tep4-8k1k.yaml +++ /dev/null @@ -1,73 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-4p1d-tp1-tep4-fp4-8k1k" -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 0 - prefill_workers: 4 - decode_workers: 1 - gpus_per_prefill: 1 - gpus_per_decode: 4 -dynamo: - install: true - version: 1.3.0.dev20260713 -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: - allow_prefill_decode_colocation: true - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - enforce-eager: true - max-num-batched-tokens: 8192 - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 -health_check: - max_attempts: 360 - interval_seconds: 10 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/4p2d-dep2-tep4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/4p2d-dep2-tep4-8k1k.yaml deleted file mode 100644 index 23c99d3282..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/4p2d-dep2-tep4-8k1k.yaml +++ /dev/null @@ -1,85 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-4p2d-fp4-dep2-tep4-8k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 4 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/5p2d-tp1-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/5p2d-tp1-dep8-8k1k.yaml deleted file mode 100644 index 779a0abae7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/5p2d-tp1-dep8-8k1k.yaml +++ /dev/null @@ -1,74 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-5p2d-tp1-dep8-fp4-8k1k" -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 5 - decode_workers: 2 - gpus_per_prefill: 1 - gpus_per_decode: 8 -dynamo: - install: true - version: 1.3.0.dev20260713 -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - enforce-eager: true - max-num-batched-tokens: 8192 - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 -health_check: - max_attempts: 360 - interval_seconds: 10 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "512" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/8p2d-tp1-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/8p2d-tp1-dep8-8k1k.yaml deleted file mode 100644 index 161413ea29..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/8p2d-tp1-dep8-8k1k.yaml +++ /dev/null @@ -1,74 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-8p2d-tp1-dep8-fp4-8k1k" -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: "fp4" -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 8 - decode_workers: 2 - gpus_per_prefill: 1 - gpus_per_decode: 8 -dynamo: - install: true - version: 1.3.0.dev20260713 -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - enforce-eager: true - max-num-batched-tokens: 8192 - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 -health_check: - max_attempts: 360 - interval_seconds: 10 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "768x1024" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p1d-dep2-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p1d-dep2-tp4-eagle3-8k1k.yaml deleted file mode 100644 index 4269a63c8d..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p1d-dep2-tp4-eagle3-8k1k.yaml +++ /dev/null @@ -1,94 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p1d-dep2-tp4-fp4-8k1k-eagle3" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: fp4 - -resources: - gpu_type: b300 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 0 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - allow_prefill_decode_colocation_across_nodes: true - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - kv-cache-dtype: fp8 - gpu-memory-utilization: 0.95 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - block-size: 128 - kv-cache-dtype: fp8 - gpu-memory-utilization: 0.95 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - req_rate: inf - num_warmup_mult: 0 - random_range_ratio: 0.8 - use_chat_template: true - concurrencies: "64" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p2d-dep2-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p2d-dep2-tp4-eagle3-8k1k.yaml deleted file mode 100644 index 7cedc791e0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p2d-dep2-tp4-eagle3-8k1k.yaml +++ /dev/null @@ -1,94 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p2d-dep2-tp4-fp4-8k1k-eagle3" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: fp4 - -resources: - gpu_type: b300 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - allow_prefill_decode_colocation_across_nodes: true - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - kv-cache-dtype: fp8 - gpu-memory-utilization: 0.95 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - block-size: 128 - kv-cache-dtype: fp8 - gpu-memory-utilization: 0.95 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - req_rate: inf - num_warmup_mult: 0 - random_range_ratio: 0.8 - use_chat_template: true - concurrencies: "4x32x64" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p4d-dep2-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p4d-dep2-tp4-eagle3-8k1k.yaml deleted file mode 100644 index 843269fa26..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p4d-dep2-tp4-eagle3-8k1k.yaml +++ /dev/null @@ -1,94 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p4d-dep2-tp4-fp4-8k1k-eagle3" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: fp4 - -resources: - gpu_type: b300 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - allow_prefill_decode_colocation_across_nodes: true - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - kv-cache-dtype: fp8 - gpu-memory-utilization: 0.95 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - block-size: 128 - kv-cache-dtype: fp8 - gpu-memory-utilization: 0.95 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - req_rate: inf - num_warmup_mult: 0 - random_range_ratio: 0.8 - use_chat_template: true - concurrencies: "4x8x64" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p6d-dep2-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p6d-dep2-tp4-eagle3-8k1k.yaml deleted file mode 100644 index a7101ec777..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p6d-dep2-tp4-eagle3-8k1k.yaml +++ /dev/null @@ -1,94 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p6d-dep2-tp4-fp4-8k1k-eagle3" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: fp4 - -resources: - gpu_type: b300 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 3 - prefill_workers: 1 - decode_workers: 6 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - allow_prefill_decode_colocation_across_nodes: true - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - kv-cache-dtype: fp8 - gpu-memory-utilization: 0.95 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - block-size: 128 - kv-cache-dtype: fp8 - gpu-memory-utilization: 0.95 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - req_rate: inf - num_warmup_mult: 0 - random_range_ratio: 0.8 - use_chat_template: true - concurrencies: "24x48" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/2p1d-dep2-dep4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/2p1d-dep2-dep4-eagle3-8k1k.yaml deleted file mode 100644 index 8187c94cd7..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/2p1d-dep2-dep4-eagle3-8k1k.yaml +++ /dev/null @@ -1,98 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p1d-dep2-dep4-fp4-8k1k-eagle3" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: fp4 - -resources: - gpu_type: b300 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 0 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - allow_prefill_decode_colocation_across_nodes: true - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm - UCX_TLS: cuda_copy,cuda_ipc,rc - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - VLLM_FLASHINFER_ALLREDUCE_BACKEND: trtllm - UCX_TLS: cuda_copy,cuda_ipc,rc - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - kv-cache-dtype: fp8 - gpu-memory-utilization: 0.95 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - block-size: 128 - kv-cache-dtype: fp8 - gpu-memory-utilization: 0.9 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - req_rate: inf - num_warmup_mult: 0 - random_range_ratio: 0.8 - use_chat_template: true - concurrencies: "256x512" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/2p3d-dep2-tp4-eagle3-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/2p3d-dep2-tp4-eagle3-8k1k.yaml deleted file mode 100644 index 376503040a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/2p3d-dep2-tp4-eagle3-8k1k.yaml +++ /dev/null @@ -1,94 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p3d-dep2-tp4-fp4-8k1k-eagle3" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" - precision: fp4 - -resources: - gpu_type: b300 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 2 - decode_workers: 3 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - allow_prefill_decode_colocation_across_nodes: true - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: cuda_copy,cuda_ipc,rc - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - block-size: 128 - kv-cache-dtype: fp8 - gpu-memory-utilization: 0.95 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":1,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' - block-size: 128 - kv-cache-dtype: fp8 - gpu-memory-utilization: 0.95 - max-model-len: 9472 - language-model-only: true - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - req_rate: inf - num_warmup_mult: 0 - random_range_ratio: 0.8 - use_chat_template: true - concurrencies: "192" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p1d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p1d-dep2-tep8-1k1k.yaml deleted file mode 100644 index 5bbb133628..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p1d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,79 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p1d-fp8-dep2-tep8-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4x16x64x128x4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p1d-dep2-tp4-marlin-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p1d-dep2-tp4-marlin-1k1k.yaml deleted file mode 100644 index 49a60981ee..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p1d-dep2-tp4-marlin-1k1k.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p1d-fp8-dep2-tp4-marlin-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - - prefill_environment: - UCX_TLS: "cuda_ipc,cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_ipc,cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - tensor-parallel-size: 4 - enable-expert-parallel: false - moe-backend: marlin - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1x4x8x16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p2d-dep2-dep4-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p2d-dep2-dep4-1k1k.yaml deleted file mode 100644 index ef7e66d765..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p2d-dep2-dep4-1k1k.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p2d-fp8-dep2-dep4-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p1d-dep2-dep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p1d-dep2-dep8-1k1k.yaml deleted file mode 100644 index 9f5aa341cf..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p1d-dep2-dep8-1k1k.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p1d-fp8-dep2-dep8-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "512x4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p1d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p1d-dep2-tep8-1k1k.yaml deleted file mode 100644 index 42c6e7bbc2..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p1d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,79 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p1d-fp8-dep2-tep8-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "32" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p2d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p2d-dep2-tep8-1k1k.yaml deleted file mode 100644 index 3e701df05e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p2d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,79 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p2d-fp8-dep2-tep8-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/3p2d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/3p2d-dep2-tep8-1k1k.yaml deleted file mode 100644 index b9a1d10588..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/3p2d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,79 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-3p2d-fp8-dep2-tep8-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 3 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p1d-dep2-tp4-marlin-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p1d-dep2-tp4-marlin-8k1k.yaml deleted file mode 100644 index a51cfbd2a3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p1d-dep2-tp4-marlin-8k1k.yaml +++ /dev/null @@ -1,85 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p1d-fp8-dep2-tp4-marlin-8k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_ipc,cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_ipc,cuda_copy,rc" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - tensor-parallel-size: 4 - enable-expert-parallel: false - moe-backend: marlin - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1x4x8x16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p2d-dep2-tep4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p2d-dep2-tep4-8k1k.yaml deleted file mode 100644 index a962183696..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p2d-dep2-tep4-8k1k.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p2d-fp8-dep2-tep4-8k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - tensor-parallel-size: 4 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 512 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p4d-dep2-tep4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p4d-dep2-tep4-8k1k.yaml deleted file mode 100644 index a41361ae66..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p4d-dep2-tep4-8k1k.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p4d-fp8-dep2-tep4-8k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - tensor-parallel-size: 4 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 512 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "16x32x64x128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p4d-dep2-tep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p4d-dep2-tep8-8k1k.yaml deleted file mode 100644 index c1a97a0c07..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p4d-dep2-tep8-8k1k.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p4d-fp8-dep2-tep8-8k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/2p2d-dep2-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/2p2d-dep2-dep8-8k1k.yaml deleted file mode 100644 index 873271d90a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/2p2d-dep2-dep8-8k1k.yaml +++ /dev/null @@ -1,85 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p2d-fp8-dep2-dep8-8k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256x512" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/2p2d-dep2-tep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/2p2d-dep2-tep8-8k1k.yaml deleted file mode 100644 index 71fcc81d96..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/2p2d-dep2-tep8-8k1k.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p2d-fp8-dep2-tep8-8k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/3p2d-dep2-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/3p2d-dep2-dep8-8k1k.yaml deleted file mode 100644 index babde558e3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/3p2d-dep2-dep8-8k1k.yaml +++ /dev/null @@ -1,85 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-3p2d-fp8-dep2-dep8-8k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 3 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 # Per DP rank: 2 workers x DP8 = 16 ranks. - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1024x2048" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/4p2d-dep2-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/4p2d-dep2-dep8-8k1k.yaml deleted file mode 100644 index e36f12dfa5..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/4p2d-dep2-dep8-8k1k.yaml +++ /dev/null @@ -1,85 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-4p2d-fp8-dep2-dep8-8k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 4 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260710 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 # Per DP rank: 2 workers x DP8 = 16 ranks. - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml deleted file mode 100644 index faf3d780be..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml +++ /dev/null @@ -1,91 +0,0 @@ -name: "minimax-m3-vllm-agg-gb200-dep4-agentic" - -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.1" } -dynamo: { version: "1.3.1", install: true } -environment: { ETCD_LEASE_TTL: "7200" } -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } -frontend: - type: dynamo - enable_multiple_frontends: false - env: { DYN_TCP_REQUEST_TIMEOUT: "60" } - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-reset-states: true - router-temperature: "0" - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 128 -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: { aggregated: true } - aggregated_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - vllm_config: - aggregated: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml deleted file mode 100644 index 3d81cbacf0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml +++ /dev/null @@ -1,93 +0,0 @@ -name: "minimax-m3-vllm-agg-gb200-dep4-vllm-simple-agentic" - -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.1" } -dynamo: { version: "1.3.1", install: true } -environment: { ETCD_LEASE_TTL: "7200" } -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } -frontend: - type: dynamo - enable_multiple_frontends: false - env: { DYN_TCP_REQUEST_TIMEOUT: "60" } - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-reset-states: true - router-temperature: "0" - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 128 -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: { aggregated: true } - aggregated_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_SIMPLE_KV_OFFLOAD: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - vllm_config: - aggregated: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml deleted file mode 100644 index c056c101dd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml +++ /dev/null @@ -1,91 +0,0 @@ -name: "minimax-m3-vllm-agg-gb200-dep8-agentic" - -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.1" } -dynamo: { version: "1.3.1", install: true } -environment: { ETCD_LEASE_TTL: "7200" } -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 2, agg_workers: 1, gpus_per_agg: 8 } -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } -frontend: - type: dynamo - enable_multiple_frontends: false - env: { DYN_TCP_REQUEST_TIMEOUT: "60" } - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-reset-states: true - router-temperature: "0" - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 128 -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: { aggregated: true } - aggregated_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - vllm_config: - aggregated: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic-nightly-native.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic-nightly-native.yaml deleted file mode 100644 index 83a4e51f55..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic-nightly-native.yaml +++ /dev/null @@ -1,100 +0,0 @@ -name: "minimax-m3-vllm-agg-gb200-tp4-agentic-nightly-native" - -model: - path: "minimax-m3-nvfp4" - container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" - precision: "fp4" - -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" } - frameworks: { dynamo: "1.5.0.dev20260908" } - -dynamo: { version: "1.5.0.dev20260908", install: true } -environment: { ETCD_LEASE_TTL: "7200" } - -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } - -frontend: - type: dynamo - enable_multiple_frontends: false - env: { DYN_TCP_REQUEST_TIMEOUT: "60" } - args: - trust-remote-code: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 128 - -backend: - type: vllm - connector: null - kv_events_config: { aggregated: true } - aggregated_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - vllm_config: - aggregated: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } - -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml deleted file mode 100644 index 6403d69e5b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml +++ /dev/null @@ -1,104 +0,0 @@ -name: "minimax-m3-vllm-agg-gb200-tp4-agentic" - -model: - path: "minimax-m3-nvfp4" - container: "vllm/vllm-openai:v0.27.1" - precision: "fp4" - -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.1" } - -dynamo: { version: "1.3.1", install: true } -environment: { ETCD_LEASE_TTL: "7200" } - -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } - -frontend: - type: dynamo - enable_multiple_frontends: false - env: { DYN_TCP_REQUEST_TIMEOUT: "60" } - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-reset-states: true - router-temperature: "0" - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 128 - -backend: - type: vllm - connector: null - kv_events_config: { aggregated: true } - aggregated_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - vllm_config: - aggregated: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic-nightly-native.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic-nightly-native.yaml deleted file mode 100644 index 327af14f6c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic-nightly-native.yaml +++ /dev/null @@ -1,94 +0,0 @@ -name: "minimax-m3-vllm-agg-gb200-tp4-vllm-simple-agentic-nightly-native" - -model: - path: "minimax-m3-nvfp4" - container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" - precision: "fp4" - -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" } - frameworks: { dynamo: "1.5.0.dev20260908" } - -dynamo: { version: "1.5.0.dev20260908", install: true } -environment: { ETCD_LEASE_TTL: "7200" } -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } - -frontend: - type: dynamo - enable_multiple_frontends: false - env: { DYN_TCP_REQUEST_TIMEOUT: "60" } - args: - trust-remote-code: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 128 - -backend: - type: vllm - connector: null - kv_events_config: { aggregated: true } - aggregated_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_SIMPLE_KV_OFFLOAD: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - vllm_config: - aggregated: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":777389080576,"cpu_bytes_to_use_per_rank":194347270144,"lazy_offload":true}}' - stream-interval: 20 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } - -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml deleted file mode 100644 index 41a1835083..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml +++ /dev/null @@ -1,98 +0,0 @@ -name: "minimax-m3-vllm-agg-gb200-tp4-vllm-simple-agentic" - -model: - path: "minimax-m3-nvfp4" - container: "vllm/vllm-openai:v0.27.1" - precision: "fp4" - -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.1" } - -dynamo: { version: "1.3.1", install: true } -environment: { ETCD_LEASE_TTL: "7200" } -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } - -frontend: - type: dynamo - enable_multiple_frontends: false - env: { DYN_TCP_REQUEST_TIMEOUT: "60" } - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-reset-states: true - router-temperature: "0" - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 128 - -backend: - type: vllm - connector: null - kv_events_config: { aggregated: true } - aggregated_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_SIMPLE_KV_OFFLOAD: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - vllm_config: - aggregated: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' - stream-interval: 20 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp8-agentic-nightly-native.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp8-agentic-nightly-native.yaml deleted file mode 100644 index 4dec81758a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp8-agentic-nightly-native.yaml +++ /dev/null @@ -1,100 +0,0 @@ -name: "minimax-m3-vllm-agg-gb200-tp8-agentic-nightly-native" - -model: - path: "minimax-m3-nvfp4" - container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" - precision: "fp4" - -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" } - frameworks: { dynamo: "1.5.0.dev20260908" } - -dynamo: { version: "1.5.0.dev20260908", install: true } -environment: { ETCD_LEASE_TTL: "7200" } - -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } - -frontend: - type: dynamo - enable_multiple_frontends: false - env: { DYN_TCP_REQUEST_TIMEOUT: "60" } - args: - trust-remote-code: true - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 128 - -backend: - type: vllm - connector: null - kv_events_config: { aggregated: true } - aggregated_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - vllm_config: - aggregated: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.9 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' - stream-interval: 20 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } - -benchmark: - type: custom - use_chat_template: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep8-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep8-dep4-agentic.yaml deleted file mode 100644 index 1a798085b9..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep8-dep4-agentic.yaml +++ /dev/null @@ -1,128 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p1d-dep8-dep4-agentic" - -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.1" } -dynamo: { version: "1.3.1", install: true } -environment: { ETCD_LEASE_TTL: "7200" } -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 4 -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } -frontend: - type: dynamo - enable_multiple_frontends: false - env: { DYN_TCP_REQUEST_TIMEOUT: "60" } - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "kv" - router-kv-events: true - router-reset-states: true - router-temperature: "0" - router-session-affinity-ttl-secs: 14400 - kv-cache-block-size: 128 -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: true - prefill_environment: &worker_env - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_NIXL_ABORT_REQUEST_TIMEOUT: "300" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - decode_environment: *worker_env - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - compilation-config: '{"cudagraph_mode":"PIECEWISE"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - kv-cache-metrics: true - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-tp4-tp4-c24-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-tp4-tp4-c24-agentic.yaml deleted file mode 100644 index 19c6748e90..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-tp4-tp4-c24-agentic.yaml +++ /dev/null @@ -1,159 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p1d-tp4-tp4-c24-agentic" - -model: - path: "minimax-m3-nvfp4" - container: &container "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6" - precision: "fp4" - -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: *container } - frameworks: { dynamo: "1.5.0.dev20260819" } - -dynamo: { wheel: "1.5.0.dev20260819", install: true } -environment: { PYTHONHASHSEED: "0" } -health_check: { max_attempts: 2160, interval_seconds: 10 } - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 3600 - env: - DYN_LOG: "info" - DYN_TCP_CONNECT_TIMEOUT: "120" - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE: "1" - -backend: - type: vllm - connector: null - dp_launch_mode: per_gpu - mooncake_kv_store: - env: - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "1" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - prefill_environment: &worker_environment - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MEMTYPE_CACHE: "n" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - UCX_TLS: "tcp,cuda_ipc,cuda_copy" - WITH_NVIDIA_PEERMEM: "0" - decode_environment: *worker_environment - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: &numa_nodes [0, 0, 1, 1] - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: *numa_nodes - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AGENTIC_WARMUP_GRACE_PERIOD: "1800" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "3600" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-tp4-tp8-c1-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-tp4-tp8-c1-agentic.yaml deleted file mode 100644 index 7ab2a4506a..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-tp4-tp8-c1-agentic.yaml +++ /dev/null @@ -1,162 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p1d-tp4-tp8-c1-agentic" - -model: - path: "minimax-m3-nvfp4" - container: &container "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6" - precision: "fp4" - -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: *container } - frameworks: { dynamo: "1.5.0.dev20260819" } - -dynamo: { wheel: "1.5.0.dev20260819", install: true } -environment: { PYTHONHASHSEED: "0" } -health_check: { max_attempts: 2160, interval_seconds: 10 } - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 3600 - env: - DYN_LOG: "info" - DYN_TCP_CONNECT_TIMEOUT: "120" - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE: "1" - -backend: - type: vllm - connector: null - dp_launch_mode: per_gpu - mooncake_kv_store: - env: - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "1" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - prefill_environment: &worker_environment - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MEMTYPE_CACHE: "n" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - UCX_TLS: "tcp,cuda_ipc,cuda_copy" - WITH_NVIDIA_PEERMEM: "0" - decode_environment: - <<: *worker_environment - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: &numa_nodes [0, 0, 1, 1] - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 8 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-num-seqs: 1 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: *numa_nodes - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AGENTIC_WARMUP_GRACE_PERIOD: "1800" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "3600" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p2d-tp4-tp4-c8-c16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p2d-tp4-tp4-c8-c16-agentic.yaml deleted file mode 100644 index 562a44fcf2..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p2d-tp4-tp4-c8-c16-agentic.yaml +++ /dev/null @@ -1,159 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p2d-tp4-tp4-c8-c16-agentic" - -model: - path: "minimax-m3-nvfp4" - container: &container "vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6" - precision: "fp4" - -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: *container } - frameworks: { dynamo: "1.5.0.dev20260819" } - -dynamo: { wheel: "1.5.0.dev20260819", install: true } -environment: { PYTHONHASHSEED: "0" } -health_check: { max_attempts: 2160, interval_seconds: 10 } - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 3600 - env: - DYN_LOG: "info" - DYN_TCP_CONNECT_TIMEOUT: "120" - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE: "1" - -backend: - type: vllm - connector: null - dp_launch_mode: per_gpu - mooncake_kv_store: - env: - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "1" - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - prefill_environment: &worker_environment - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - DYN_TCP_CONNECT_TIMEOUT: "120" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MEMTYPE_CACHE: "n" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - UCX_TLS: "tcp,cuda_ipc,cuda_copy" - WITH_NVIDIA_PEERMEM: "0" - decode_environment: *worker_environment - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: &numa_nodes [0, 0, 1, 1] - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 4 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 128 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: *numa_nodes - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.78}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AGENTIC_WARMUP_GRACE_PERIOD: "1800" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "3600" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt_eval.sh b/benchmarks/multi_node/srt_eval.sh new file mode 100644 index 0000000000..c83ef10a6a --- /dev/null +++ b/benchmarks/multi_node/srt_eval.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 SemiAnalysis LLC. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Accuracy evaluation using InferenceX benchmark_lib. +# Requires: endpoint infmax_workspace and explicit workflow metadata. + +set -e + +if [[ $# -ne 2 || -z "$1" || -z "$2" ]]; then + echo "Usage: $0 endpoint infmax_workspace" >&2 + exit 1 +fi + +ENDPOINT=$1 +INFMAX_WORKSPACE=$2 + +# Extract HOST and PORT from endpoint (e.g., http://localhost:8000) +HOST=$(echo "$ENDPOINT" | sed -E 's|https?://||; s|:.*||') +PORT=$(echo "$ENDPOINT" | sed -E 's|.*:([0-9]+).*|\1|') + +echo "Eval Config: endpoint=${ENDPOINT}; host=${HOST}; port=${PORT}; workspace=${INFMAX_WORKSPACE}" + +# cd to workspace so that relative paths (e.g., utils/evals/*.yaml) resolve +cd "${INFMAX_WORKSPACE}" + +# Source the InferenceX benchmark library +source "${INFMAX_WORKSPACE}/benchmarks/benchmark_lib.sh" + +# The workflow supplies topology and concurrency; srt-slurm supplies MODEL_NAME +# from the recipe's served model name. Missing inputs are configuration errors. +check_env_vars IS_MULTINODE MODEL_NAME EVAL_CONC PREFILL_TP PREFILL_EP PREFILL_DP_ATTN DECODE_DP_ATTN + +# Translate the explicit workflow names to benchmark_lib's metadata names. +export EVAL_CONCURRENT_REQUESTS="$EVAL_CONC" +export TP="$PREFILL_TP" +export CONC="$EVAL_CONC" +export EP_SIZE="$PREFILL_EP" +export DP_ATTENTION="$PREFILL_DP_ATTN" +export PREFILL_DP_ATTENTION="$PREFILL_DP_ATTN" +export DECODE_DP_ATTENTION="$DECODE_DP_ATTN" + +echo "Running evaluation for ${MODEL_NAME} with concurrent-requests=${EVAL_CONCURRENT_REQUESTS}..." +eval_rc=0 +run_eval --port "$PORT" || eval_rc=$? + +# Generate the lm-eval summary +echo "Generating lm-eval summary..." +append_lm_eval_summary || true + +# Copy eval artifacts to /logs/eval_results/ +mkdir -p /logs/eval_results +echo "Copying eval artifacts to /logs/eval_results/..." +cp -v meta_env.json /logs/eval_results/ 2>/dev/null || true +stage_eval_artifacts /logs/eval_results "$PWD" || true + +if [[ "$eval_rc" -ne 0 ]]; then + echo "Evaluation failed with exit code ${eval_rc}" + exit "$eval_rc" +fi + +echo "Evaluation complete" diff --git a/configs/deprecated/nvidia-1k1k-master.yaml b/configs/deprecated/nvidia-1k1k-master.yaml index f0c5df337d..48feaa0c28 100644 --- a/configs/deprecated/nvidia-1k1k-master.yaml +++ b/configs/deprecated/nvidia-1k1k-master.yaml @@ -1485,7 +1485,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b1-eplb0-mtp0.yaml" decode: num-worker: 4 tp: 8 @@ -1499,7 +1499,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 4 @@ -1513,7 +1513,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b4-eplb0-mtp0.yaml" decode: num-worker: 4 tp: 8 @@ -1527,7 +1527,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 4 @@ -1541,7 +1541,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch16_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b16-eplb0-mtp0.yaml" decode: num-worker: 4 tp: 8 @@ -1555,7 +1555,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch32_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b32-eplb0-mtp0.yaml" decode: num-worker: 4 tp: 8 @@ -1569,7 +1569,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch64_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b64-eplb0-mtp0.yaml" decode: num-worker: 4 tp: 8 @@ -1583,7 +1583,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch16_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b16-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 32 @@ -1597,7 +1597,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch32_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b32-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 32 @@ -1611,7 +1611,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch64_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep32-b64-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 32 @@ -1625,7 +1625,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch128_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep16-b128-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 16 @@ -1639,7 +1639,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch256_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep16-b256-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 16 @@ -1653,7 +1653,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep8_batch512_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep8_batch512_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep8-b512-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 8 @@ -1685,7 +1685,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b1-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -1700,7 +1700,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp3.yaml" decode: num-worker: 5 tp: 4 @@ -1715,7 +1715,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch4_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b4-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -1730,7 +1730,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch8_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b8-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -1745,7 +1745,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch16_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tep8-b16-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -1760,7 +1760,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b4-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -1775,7 +1775,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b8-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -1790,7 +1790,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch32_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep16-b32-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 16 @@ -1805,7 +1805,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep32-b16-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -1820,7 +1820,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep16_batch64_eplb0_mtp1.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep16-b64-eplb0-mtp1.yaml" decode: num-worker: 1 tp: 16 @@ -1835,7 +1835,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch128_eplb0_mtp1.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep16-b128-eplb0-mtp1.yaml" decode: num-worker: 1 tp: 16 @@ -1850,7 +1850,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep16_batch256_eplb256_mtp1.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep16-b256-eplb256-mtp1.yaml" decode: num-worker: 1 tp: 16 @@ -1865,7 +1865,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch512_eplb0_mtp1.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/1k1k/disagg-2p1d-dep4-dep8-b512-eplb0-mtp1.yaml" decode: num-worker: 1 tp: 8 @@ -4488,7 +4488,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p5d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c16x512-mtp.yaml" decode: num-worker: 5 tp: 8 @@ -4502,7 +4502,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp8-ep8-b512-c32x64x256x512-mtp.yaml" decode: num-worker: 6 tp: 8 @@ -4516,7 +4516,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_0.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b1024-c512x1024-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -4530,7 +4530,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_1.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/1k1k/disagg-1p2d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b512-c512-mtp.yaml" decode: num-worker: 2 tp: 8 @@ -4945,7 +4945,7 @@ kimik2.5-fp4-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p1d-dep4-dep8.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep8-stp.yaml" decode: num-worker: 1 tp: 1 @@ -4958,7 +4958,7 @@ kimik2.5-fp4-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p4d-dep4-tp8.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb200-fp4/1k1k/disagg-1p4d-dep4-tp8-stp.yaml" decode: num-worker: 4 tp: 8 @@ -4971,7 +4971,7 @@ kimik2.5-fp4-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p1d-dep4-dep16.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb200-fp4/1k1k/disagg-1p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 1 @@ -5002,7 +5002,7 @@ qwen3.5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-tp4-tp4.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml" decode: num-worker: 1 tp: 4 @@ -5017,7 +5017,7 @@ qwen3.5-fp8-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-dep4-dep16.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 16 @@ -5032,7 +5032,7 @@ qwen3.5-fp8-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/1k1k/2p1d-dep4-dep16.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 16 @@ -5063,7 +5063,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[0]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_1k1k_hightpt[0]" decode: num-worker: 1 tp: 16 @@ -5078,7 +5078,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[1]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_1k1k_hightpt[1]" decode: num-worker: 1 tp: 16 @@ -5093,7 +5093,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[2]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_1k1k_hightpt[2]" decode: num-worker: 1 tp: 16 @@ -5108,7 +5108,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[3]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_1k1k_hightpt[3]" decode: num-worker: 1 tp: 32 @@ -5127,7 +5127,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_lowlat[0]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_1k1k_lowlat[0]" decode: num-worker: 16 tp: 4 @@ -5142,7 +5142,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_lowlat[1]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_1k1k_lowlat[1]" decode: num-worker: 16 tp: 4 @@ -5157,7 +5157,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_lowlat[2]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_1k1k_lowlat[2]" decode: num-worker: 16 tp: 4 @@ -5749,7 +5749,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_maxtpt_0.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-2p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b2048-c3160-stp.yaml" decode: num-worker: 1 tp: 32 @@ -5767,7 +5767,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_0.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2023-c2115-stp.yaml" decode: num-worker: 1 tp: 16 @@ -5781,7 +5781,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_1.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b1024-c1156-stp.yaml" decode: num-worker: 1 tp: 32 @@ -5795,7 +5795,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_2.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b512-c556-stp.yaml" decode: num-worker: 1 tp: 32 @@ -5809,7 +5809,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_3.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b1-c23-stp.yaml" decode: num-worker: 16 tp: 4 @@ -5823,7 +5823,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_4.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b16-c290-stp.yaml" decode: num-worker: 16 tp: 4 @@ -5837,7 +5837,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/1k1k/disagg/stp/1k1k_stp_lowlat_5.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/1k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b4-c73-stp.yaml" decode: num-worker: 16 tp: 4 @@ -5963,7 +5963,7 @@ glm5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_hightpt_0.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-2p1d-p-tp8-dp8-d-tp32-ep32-dp32-b1024-c2161-stp.yaml" decode: num-worker: 1 tp: 32 @@ -5981,7 +5981,7 @@ glm5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_0.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p1d-p-tp8-dp8-d-tp16-ep16-dp16-b1815-c1955-stp.yaml" decode: num-worker: 1 tp: 16 @@ -5995,7 +5995,7 @@ glm5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_1.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p1d-p-tp8-dp8-d-tp16-ep16-dp16-b1024-c1170-stp.yaml" decode: num-worker: 1 tp: 16 @@ -6009,7 +6009,7 @@ glm5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_2.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p1d-p-tp8-dp8-d-tp16-ep16-dp16-b256-c298-stp.yaml" decode: num-worker: 1 tp: 16 @@ -6023,7 +6023,7 @@ glm5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_3.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b1-c8-stp.yaml" decode: num-worker: 6 tp: 8 @@ -6037,7 +6037,7 @@ glm5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_4.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b16-c72-stp.yaml" decode: num-worker: 6 tp: 8 @@ -6051,7 +6051,7 @@ glm5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/1k1k/disagg/stp/1k1k_stp_lowlat_5.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/1k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b4-c20-stp.yaml" decode: num-worker: 6 tp: 8 @@ -6335,7 +6335,7 @@ glm5-fp8-gb300-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_3.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb300-fp8/1k1k/disagg-6p1d-p-tp4-dp4-d-tp48-ep48-dp48-b6144-c6500-mtp.yaml" decode: num-worker: 1 tp: 48 @@ -6349,7 +6349,7 @@ glm5-fp8-gb300-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_hightpt_4.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb300-fp8/1k1k/disagg-4p1d-p-tp4-dp4-d-tp56-ep56-dp56-b5600-c5700-mtp.yaml" decode: num-worker: 1 tp: 56 @@ -6367,7 +6367,7 @@ glm5-fp8-gb300-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb300-fp8/1k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b32-c512x256x128x64-mtp.yaml" decode: num-worker: 17 tp: 4 @@ -6381,7 +6381,7 @@ glm5-fp8-gb300-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb300-fp8/1k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b1-c32-mtp.yaml" decode: num-worker: 17 tp: 4 @@ -6418,7 +6418,7 @@ minimaxm3-fp8-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/1k1k/1p1d-dep2-tep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -6431,7 +6431,7 @@ minimaxm3-fp8-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/1k1k/1p1d-dep2-tp4-marlin-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml" decode: num-worker: 1 tp: 4 @@ -6444,7 +6444,7 @@ minimaxm3-fp8-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/1k1k/1p2d-dep2-dep4-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-1p2d-dep2-dep4-stp.yaml" decode: num-worker: 2 tp: 4 @@ -6457,7 +6457,7 @@ minimaxm3-fp8-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/1k1k/2p1d-dep2-dep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-dep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -6470,7 +6470,7 @@ minimaxm3-fp8-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/1k1k/2p1d-dep2-tep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p1d-dep2-tep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -6483,7 +6483,7 @@ minimaxm3-fp8-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/1k1k/2p2d-dep2-tep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -6496,7 +6496,7 @@ minimaxm3-fp8-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/1k1k/3p2d-dep2-tep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp8/1k1k/disagg-3p2d-dep2-tep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -6525,7 +6525,7 @@ minimaxm3-fp4-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/1k1k/1p1d-dep2-tep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -6538,7 +6538,7 @@ minimaxm3-fp4-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/1k1k/1p1d-dep2-tp4-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p1d-dep2-tp4-stp.yaml" decode: num-worker: 1 tp: 4 @@ -6551,7 +6551,7 @@ minimaxm3-fp4-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/1k1k/1p2d-dep2-dep4-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-1p2d-dep2-dep4-stp.yaml" decode: num-worker: 2 tp: 4 @@ -6564,7 +6564,7 @@ minimaxm3-fp4-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/1k1k/2p1d-dep2-dep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-dep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -6577,7 +6577,7 @@ minimaxm3-fp4-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/1k1k/2p1d-dep2-tep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p1d-dep2-tep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -6590,7 +6590,7 @@ minimaxm3-fp4-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/1k1k/2p2d-dep2-tep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-2p2d-dep2-tep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -6603,7 +6603,7 @@ minimaxm3-fp4-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/1k1k/3p2d-dep2-tep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/1k1k/disagg-3p2d-dep2-tep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -6635,7 +6635,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/1k1k/1p1d-dep2-dep4-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p1d-dep2-dep4-stp.yaml" decode: num-worker: 1 tp: 4 @@ -6650,7 +6650,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/1k1k/1p2d-dep2-tep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-1p2d-dep2-tep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -6665,7 +6665,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p2d-dep2-tep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p2d-dep2-tep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -6680,7 +6680,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p3d-dep2-dep4-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p3d-dep2-dep4-stp.yaml" decode: num-worker: 3 tp: 4 @@ -6695,7 +6695,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p4d-dep2-dep4-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-2p4d-dep2-dep4-stp.yaml" decode: num-worker: 4 tp: 4 @@ -6710,7 +6710,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/1k1k/4p2d-dep2-dep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/1k1k/disagg-4p2d-dep2-dep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -6759,7 +6759,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p1d-dep4-dep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-dep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -6774,7 +6774,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p1d-dep4-tep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p1d-dep4-tep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -6789,7 +6789,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p2d-dep4-dep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-dep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -6804,7 +6804,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p2d-dep4-tep8-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p2d-dep4-tep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -6819,7 +6819,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p4d-dep4-tp4-marlin-1k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp8/1k1k/disagg-1p4d-dep4-tp4-marlin-stp.yaml" decode: num-worker: 4 tp: 4 @@ -7051,7 +7051,7 @@ kimik2.5-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-1p1d-dep4-dep16.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-1p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 1 @@ -7064,7 +7064,7 @@ kimik2.5-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-1p1d-dep4-dep24.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-1p1d-dep4-dep24-stp.yaml" decode: num-worker: 1 tp: 1 @@ -7077,7 +7077,7 @@ kimik2.5-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-1p2d-dep4-dep4.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-1p2d-dep4-dep4-stp.yaml" decode: num-worker: 2 tp: 1 @@ -7090,7 +7090,7 @@ kimik2.5-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-1p7d-tep4-tp4.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-1p7d-tep4-tp4-stp.yaml" decode: num-worker: 7 tp: 4 @@ -7103,7 +7103,7 @@ kimik2.5-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb300-2p3d-dep4-dep8.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb300-fp4/1k1k/disagg-2p3d-dep4-dep8-stp.yaml" decode: num-worker: 3 tp: 1 @@ -7316,7 +7316,7 @@ qwen3.5-fp8-gb300-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-tp4-tp4.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-tp4-tp4-stp.yaml" decode: num-worker: 1 tp: 4 @@ -7331,7 +7331,7 @@ qwen3.5-fp8-gb300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-dep4-dep16.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-1p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 16 @@ -7346,7 +7346,7 @@ qwen3.5-fp8-gb300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/1k1k/2p1d-dep4-dep16.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/1k1k/disagg-2p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 16 diff --git a/configs/deprecated/nvidia-glm5-glm5.1-master.yaml b/configs/deprecated/nvidia-glm5-glm5.1-master.yaml index 9c275f3ed1..f7a55f4676 100644 --- a/configs/deprecated/nvidia-glm5-glm5.1-master.yaml +++ b/configs/deprecated/nvidia-glm5-glm5.1-master.yaml @@ -309,7 +309,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 4 @@ -323,7 +323,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b2-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 4 @@ -337,7 +337,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 4 @@ -351,7 +351,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b8-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 4 @@ -365,7 +365,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b16-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 4 @@ -379,7 +379,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep4-dep32-b8-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 32 @@ -393,7 +393,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-4p1d-dep4-dep32-b16-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 32 @@ -407,7 +407,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-5p1d-dep4-dep16-b64-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 16 @@ -421,7 +421,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-6p1d-dep4-dep32-b32-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 32 @@ -435,7 +435,7 @@ glm5-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-9p1d-dep4-dep16-b128-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 16 @@ -469,7 +469,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp3.yaml" decode: num-worker: 5 tp: 4 @@ -484,7 +484,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b2-eplb0-mtp3.yaml" decode: num-worker: 5 tp: 4 @@ -499,7 +499,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp3.yaml" decode: num-worker: 5 tp: 4 @@ -514,7 +514,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-1p1d-dep4-dep32-b2-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -529,7 +529,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch4_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep4-dep32-b4-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -544,7 +544,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-4p1d-dep4-dep32-b8-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -559,7 +559,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-6p1d-dep4-dep16-b32-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 16 @@ -574,7 +574,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep16_batch64_eplb0_mtp1.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep4-dep16-b64-eplb0-mtp1.yaml" decode: num-worker: 1 tp: 16 @@ -589,7 +589,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep32_batch16_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep4-dep32-b16-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -604,7 +604,7 @@ glm5-fp4-gb200-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/v1.0.26/recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/glm5/disagg/trtllm_dynamo/gb200_nvfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep16_batch128_eplb0_mtp1.yaml" + - "CONFIG_FILE=recipes/glm5/trtllm/gb200-fp4/8k1k/disagg-12p1d-dep4-dep16-b128-eplb0-mtp1.yaml" decode: num-worker: 1 tp: 16 @@ -669,7 +669,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[0]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_hightpt[0]" decode: num-worker: 1 tp: 16 @@ -684,7 +684,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[1]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_hightpt[1]" decode: num-worker: 1 tp: 16 @@ -699,7 +699,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[2]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_hightpt[2]" decode: num-worker: 1 tp: 16 @@ -714,7 +714,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[3]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_hightpt[3]" decode: num-worker: 1 tp: 16 @@ -733,7 +733,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[0]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_lowlat[0]" decode: num-worker: 4 tp: 4 @@ -748,7 +748,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[1]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_lowlat[1]" decode: num-worker: 4 tp: 4 @@ -763,7 +763,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[2]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_lowlat[2]" decode: num-worker: 8 tp: 4 @@ -778,7 +778,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[3]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_lowlat[3]" decode: num-worker: 8 tp: 4 @@ -793,7 +793,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[4]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_lowlat[4]" decode: num-worker: 16 tp: 4 @@ -808,7 +808,7 @@ glm5-fp4-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[5]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_lowlat[5]" decode: num-worker: 16 tp: 4 @@ -975,7 +975,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b599-c989-stp.yaml" decode: num-worker: 1 tp: 32 @@ -989,7 +989,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b512-c686-stp.yaml" decode: num-worker: 1 tp: 32 @@ -1003,7 +1003,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b914-c1497-stp.yaml" decode: num-worker: 1 tp: 32 @@ -1017,7 +1017,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_3.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-10p1d-p-tp4-ep4-dp4-d-tp32-ep32-dp32-b1712-c2674-stp.yaml" decode: num-worker: 1 tp: 32 @@ -1035,7 +1035,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p4d-p-tp4-ep4-dp4-d-tp4-b32-c133-stp.yaml" decode: num-worker: 4 tp: 4 @@ -1049,7 +1049,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp4-b22-c146-stp.yaml" decode: num-worker: 6 tp: 4 @@ -1063,7 +1063,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp4-b16-c103-stp.yaml" decode: num-worker: 6 tp: 4 @@ -1077,7 +1077,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p8d-p-tp4-ep4-dp4-d-tp4-b15-c130-stp.yaml" decode: num-worker: 8 tp: 4 @@ -1091,7 +1091,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b6-c113-stp.yaml" decode: num-worker: 16 tp: 4 @@ -1105,7 +1105,7 @@ glm5-fp4-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_5.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp4/8k1k/disagg-1p16d-p-tp4-ep4-dp4-d-tp4-b1-c23-stp.yaml" decode: num-worker: 16 tp: 4 @@ -1185,7 +1185,7 @@ glm5-fp4-gb300-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml" + - "CONFIG_FILE=recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p3d-p-tp4-dp4-d-tp4-b128-c128-stp.yaml" router: { name: dynamo-router, version: "1.1.0" } decode: num-worker: 3 @@ -1200,7 +1200,7 @@ glm5-fp4-gb300-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml" + - "CONFIG_FILE=recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p5d-p-tp4-dp4-d-tp4-b64-c64-stp.yaml" router: { name: dynamo-router, version: "1.1.0" } decode: num-worker: 5 @@ -1215,7 +1215,7 @@ glm5-fp4-gb300-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml" + - "CONFIG_FILE=recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p9d-p-tp4-dp4-d-tp4-b32-c32-stp.yaml" router: { name: dynamo-router, version: "1.1.0" } decode: num-worker: 9 @@ -1230,7 +1230,7 @@ glm5-fp4-gb300-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml" + - "CONFIG_FILE=recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p15d-p-tp4-dp4-d-tp4-b16-c16-stp.yaml" router: { name: dynamo-router, version: "1.1.0" } decode: num-worker: 15 @@ -1245,7 +1245,7 @@ glm5-fp4-gb300-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml" + - "CONFIG_FILE=recipes/glm5/sglang/gb300-fp4/8k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b1-c12-stp.yaml" router: { name: dynamo-router, version: "1.1.0" } decode: num-worker: 17 @@ -1279,7 +1279,7 @@ glm5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_0.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-3p1d-p-tp8-dp8-d-tp32-ep32-dp32-b256-c519-stp.yaml" decode: num-worker: 1 tp: 32 @@ -1293,7 +1293,7 @@ glm5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_1.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-4p1d-p-tp8-dp8-d-tp16-ep16-dp16-b802-c1484-stp.yaml" decode: num-worker: 1 tp: 16 @@ -1307,7 +1307,7 @@ glm5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_2.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-5p1d-p-tp8-dp8-d-tp32-ep32-dp32-b828-c1688-stp.yaml" decode: num-worker: 1 tp: 32 @@ -1321,7 +1321,7 @@ glm5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_hightpt_3.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-7p1d-p-tp8-dp8-d-tp16-ep16-dp16-b1514-c2699-stp.yaml" decode: num-worker: 1 tp: 16 @@ -1339,7 +1339,7 @@ glm5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-1p8d-p-tp8-dp8-d-tp8-b1-c8-stp.yaml" decode: num-worker: 8 tp: 8 @@ -1353,7 +1353,7 @@ glm5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-1p8d-p-tp8-dp8-d-tp8-b16-c90-stp.yaml" decode: num-worker: 8 tp: 8 @@ -1367,7 +1367,7 @@ glm5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/8k1k/disagg-1p8d-p-tp8-dp8-d-tp8-b4-c9-stp.yaml" decode: num-worker: 8 tp: 8 @@ -1402,7 +1402,7 @@ glm5-fp8-gb200-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[0]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_hightpt[0]" decode: num-worker: 1 tp: 16 @@ -1417,7 +1417,7 @@ glm5-fp8-gb200-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[1]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_hightpt[1]" decode: num-worker: 1 tp: 16 @@ -1432,7 +1432,7 @@ glm5-fp8-gb200-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[2]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_hightpt[2]" decode: num-worker: 1 tp: 16 @@ -1451,7 +1451,7 @@ glm5-fp8-gb200-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[0]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_lowlat[0]" decode: num-worker: 4 tp: 8 @@ -1466,7 +1466,7 @@ glm5-fp8-gb200-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[1]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_lowlat[1]" decode: num-worker: 4 tp: 8 @@ -1481,7 +1481,7 @@ glm5-fp8-gb200-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[2]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_lowlat[2]" decode: num-worker: 8 tp: 8 @@ -1496,7 +1496,7 @@ glm5-fp8-gb200-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[3]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_lowlat[3]" decode: num-worker: 8 tp: 8 @@ -1511,7 +1511,7 @@ glm5-fp8-gb200-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp8/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[4]" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb200-fp8/fixed-seq-len/disagg-mtp-variants.yaml:zip_override_mtp_8k1k_lowlat[4]" decode: num-worker: 8 tp: 8 @@ -1800,7 +1800,7 @@ glm5-fp8-gb300-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_0.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-14p1d-p-tp4-dp4-d-tp16-ep16-dp16-b2800-c2800-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -1814,7 +1814,7 @@ glm5-fp8-gb300-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_1.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-12p1d-p-tp4-dp4-d-tp24-ep24-dp24-b1680-c1700-mtp.yaml" decode: num-worker: 1 tp: 24 @@ -1828,7 +1828,7 @@ glm5-fp8-gb300-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_2.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-10p1d-p-tp4-dp4-d-tp32-ep32-dp32-b1280-c1300-mtp.yaml" decode: num-worker: 1 tp: 32 @@ -1842,7 +1842,7 @@ glm5-fp8-gb300-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_hightpt_3.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-8p1d-p-tp4-dp4-d-tp40-ep40-dp40-b880-c900-mtp.yaml" decode: num-worker: 1 tp: 40 @@ -1860,7 +1860,7 @@ glm5-fp8-gb300-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-1p9d-p-tp4-dp4-d-tp4-b15-c150-mtp.yaml" decode: num-worker: 9 tp: 4 @@ -1874,7 +1874,7 @@ glm5-fp8-gb300-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b8-c128x64x32-mtp.yaml" decode: num-worker: 17 tp: 4 @@ -1888,7 +1888,7 @@ glm5-fp8-gb300-dynamo-sglang-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.1/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml" + - "CONFIG_FILE=recipes/glm5.1/sglang/gb300-fp8/8k1k/disagg-1p17d-p-tp4-dp4-d-tp4-b1-c24-mtp.yaml" decode: num-worker: 17 tp: 4 diff --git a/configs/deprecated/nvidia-kimik2.5-8k1k-master.yaml b/configs/deprecated/nvidia-kimik2.5-8k1k-master.yaml index 3c22a5d346..6e51ecec88 100644 --- a/configs/deprecated/nvidia-kimik2.5-8k1k-master.yaml +++ b/configs/deprecated/nvidia-kimik2.5-8k1k-master.yaml @@ -634,7 +634,7 @@ kimik2.5-fp4-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-1p4d-dep4-tep4.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-1p4d-dep4-tep4-stp.yaml" decode: num-worker: 4 tp: 4 @@ -647,7 +647,7 @@ kimik2.5-fp4-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-1p4d-dep4-tp8.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-1p4d-dep4-tp8-stp.yaml" decode: num-worker: 4 tp: 8 @@ -660,7 +660,7 @@ kimik2.5-fp4-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-3p1d-dep4-dep16.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-3p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 1 @@ -673,7 +673,7 @@ kimik2.5-fp4-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-6p1d-dep4-dep16.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-6p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 1 @@ -686,7 +686,7 @@ kimik2.5-fp4-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-8p1d-dep4-dep16.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb200-fp4/8k1k/disagg-8p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 1 @@ -716,7 +716,7 @@ kimik2.5-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-1p4d-dep4-tp8.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-1p4d-dep4-tp8-stp.yaml" decode: num-worker: 4 tp: 8 @@ -729,7 +729,7 @@ kimik2.5-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-1p8d-dep4-tp4.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-1p8d-dep4-tp4-stp.yaml" decode: num-worker: 8 tp: 4 @@ -742,7 +742,7 @@ kimik2.5-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-2p1d-dep4-dep24.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-2p1d-dep4-dep24-stp.yaml" decode: num-worker: 1 tp: 1 @@ -755,7 +755,7 @@ kimik2.5-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-3p1d-dep4-dep16.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-3p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 1 @@ -768,7 +768,7 @@ kimik2.5-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-4p1d-dep4-dep8.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep8-stp.yaml" decode: num-worker: 1 tp: 1 @@ -781,7 +781,7 @@ kimik2.5-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb300-8p1d-dep4-dep24.yaml" + - "CONFIG_FILE=recipes/kimik2.5/vllm/gb300-fp4/8k1k/disagg-8p1d-dep4-dep24-stp.yaml" decode: num-worker: 1 tp: 1 diff --git a/configs/deprecated/nvidia-minimaxm3-8k1k-master.yaml b/configs/deprecated/nvidia-minimaxm3-8k1k-master.yaml index b8adea449d..a36e76b221 100644 --- a/configs/deprecated/nvidia-minimaxm3-8k1k-master.yaml +++ b/configs/deprecated/nvidia-minimaxm3-8k1k-master.yaml @@ -62,7 +62,7 @@ minimaxm3-fp4-b200-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/1p1d-dep2-tp4-c1-c16-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p1d-dep2-tp4-c1-c16-stp.yaml" decode: num-worker: 1 tp: 4 @@ -75,7 +75,7 @@ minimaxm3-fp4-b200-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/1p2d-dep2-tp4-c64-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p2d-dep2-tp4-c64-stp.yaml" decode: num-worker: 2 tp: 4 @@ -88,7 +88,7 @@ minimaxm3-fp4-b200-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/2p2d-dep2-tp4-c128-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-2p2d-dep2-tp4-c128-stp.yaml" decode: num-worker: 2 tp: 4 @@ -101,7 +101,7 @@ minimaxm3-fp4-b200-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c256-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-3p2d-dep2-tp4-c256-stp.yaml" decode: num-worker: 2 tp: 4 @@ -114,7 +114,7 @@ minimaxm3-fp4-b200-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c512-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-3p2d-dep2-tp4-c512-stp.yaml" decode: num-worker: 2 tp: 4 @@ -127,7 +127,7 @@ minimaxm3-fp4-b200-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/4p2d-dep2-tp4-c1024-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-4p2d-dep2-tp4-c1024-stp.yaml" decode: num-worker: 2 tp: 4 @@ -160,7 +160,7 @@ minimaxm3-fp8-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/8k1k/2p2d-dep2-dep8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-2p2d-dep2-dep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -173,7 +173,7 @@ minimaxm3-fp8-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/8k1k/2p2d-dep2-tep8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-2p2d-dep2-tep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -186,7 +186,7 @@ minimaxm3-fp8-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/8k1k/4p2d-dep2-dep8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-4p2d-dep2-dep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -199,7 +199,7 @@ minimaxm3-fp8-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/8k1k/1p1d-dep2-tp4-marlin-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-1p1d-dep2-tp4-marlin-stp.yaml" decode: num-worker: 1 tp: 4 @@ -212,7 +212,7 @@ minimaxm3-fp8-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/8k1k/3p2d-dep2-dep8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-3p2d-dep2-dep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -225,7 +225,7 @@ minimaxm3-fp8-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/8k1k/1p4d-dep2-tep4-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-1p4d-dep2-tep4-stp.yaml" decode: num-worker: 4 tp: 4 @@ -238,7 +238,7 @@ minimaxm3-fp8-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/8k1k/1p2d-dep2-tep4-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-1p2d-dep2-tep4-stp.yaml" decode: num-worker: 2 tp: 4 @@ -251,7 +251,7 @@ minimaxm3-fp8-b300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/8k1k/1p4d-dep2-tep8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp8/8k1k/disagg-1p4d-dep2-tep8-stp.yaml" decode: num-worker: 4 tp: 8 @@ -284,7 +284,7 @@ minimaxm3-fp4-b300-dynamo-vllm-8k1k-tp1: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/1p1d-tp1-tp4-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p1d-tp1-tp4-stp.yaml" decode: num-worker: 1 tp: 4 @@ -297,7 +297,7 @@ minimaxm3-fp4-b300-dynamo-vllm-8k1k-tp1: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/1p1d-tp1-tp8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p1d-tp1-tp8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -310,7 +310,7 @@ minimaxm3-fp4-b300-dynamo-vllm-8k1k-tp1: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/2p1d-tp1-tp4-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-2p1d-tp1-tp4-stp.yaml" decode: num-worker: 1 tp: 4 @@ -323,7 +323,7 @@ minimaxm3-fp4-b300-dynamo-vllm-8k1k-tp1: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/3p1d-tp1-tp4-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-3p1d-tp1-tp4-stp.yaml" decode: num-worker: 1 tp: 4 @@ -336,7 +336,7 @@ minimaxm3-fp4-b300-dynamo-vllm-8k1k-tp1: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/5p2d-tp1-dep8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-5p2d-tp1-dep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -349,7 +349,7 @@ minimaxm3-fp4-b300-dynamo-vllm-8k1k-tp1: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/8p2d-tp1-dep8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-8p2d-tp1-dep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -381,7 +381,7 @@ minimaxm3-fp4-b300-dynamo-vllm-8k1k-tp1-tep4: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/4p1d-tp1-tep4-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-4p1d-tp1-tep4-stp.yaml" decode: num-worker: 1 tp: 4 @@ -413,7 +413,7 @@ minimaxm3-fp4-b300-dynamo-vllm-8k1k-legacy-max-tput: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/4p2d-dep2-tep4-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-4p2d-dep2-tep4-stp.yaml" decode: num-worker: 2 tp: 4 @@ -444,7 +444,7 @@ minimaxm3-fp4-b200-dynamo-vllm-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p6d-tp4-tp4-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p6d-tp4-tp4-eagle3-mtp.yaml" decode: num-worker: 6 tp: 4 @@ -458,7 +458,7 @@ minimaxm3-fp4-b200-dynamo-vllm-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p4d-tp4-tp4-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p4d-tp4-tp4-eagle3-mtp.yaml" decode: num-worker: 4 tp: 4 @@ -472,7 +472,7 @@ minimaxm3-fp4-b200-dynamo-vllm-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p2d-tp4-tp4-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p2d-tp4-tp4-eagle3-mtp.yaml" decode: num-worker: 2 tp: 4 @@ -486,7 +486,7 @@ minimaxm3-fp4-b200-dynamo-vllm-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p1d-tp4-tp4-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p1d-tp4-tp4-eagle3-mtp.yaml" decode: num-worker: 1 tp: 4 @@ -501,7 +501,7 @@ minimaxm3-fp4-b200-dynamo-vllm-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p3d-tp4-tp4-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p3d-tp4-tp4-eagle3-mtp.yaml" decode: num-worker: 3 tp: 4 @@ -517,7 +517,7 @@ minimaxm3-fp4-b200-dynamo-vllm-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/mtp/1p1d-tp4-dep4-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b200-fp4/8k1k/disagg-1p1d-tp4-dep4-eagle3-mtp.yaml" decode: num-worker: 1 tp: 4 @@ -548,7 +548,7 @@ minimaxm3-fp4-b300-dynamo-vllm-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p6d-dep2-tp4-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p6d-dep2-tp4-eagle3-mtp.yaml" decode: num-worker: 6 tp: 4 @@ -562,7 +562,7 @@ minimaxm3-fp4-b300-dynamo-vllm-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p4d-dep2-tp4-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p4d-dep2-tp4-eagle3-mtp.yaml" decode: num-worker: 4 tp: 4 @@ -576,7 +576,7 @@ minimaxm3-fp4-b300-dynamo-vllm-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p2d-dep2-tp4-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p2d-dep2-tp4-eagle3-mtp.yaml" decode: num-worker: 2 tp: 4 @@ -590,7 +590,7 @@ minimaxm3-fp4-b300-dynamo-vllm-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/1p1d-dep2-tp4-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-1p1d-dep2-tp4-eagle3-mtp.yaml" decode: num-worker: 1 tp: 4 @@ -605,7 +605,7 @@ minimaxm3-fp4-b300-dynamo-vllm-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/2p3d-dep2-tp4-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-2p3d-dep2-tp4-eagle3-mtp.yaml" decode: num-worker: 3 tp: 4 @@ -636,7 +636,7 @@ minimaxm3-fp4-b300-dynamo-vllm-mtp-legacy-dep4: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/mtp/2p1d-dep2-dep4-eagle3-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/b300-fp4/8k1k/disagg-2p1d-dep2-dep4-eagle3-mtp.yaml" decode: num-worker: 1 tp: 4 @@ -669,7 +669,7 @@ minimaxm3-fp8-gb300-dynamo-vllm-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c1-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp4-eagle3-c1-mtp.yaml" decode: num-worker: 1 tp: 4 @@ -683,7 +683,7 @@ minimaxm3-fp8-gb300-dynamo-vllm-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp4-eagle3-c8-mtp.yaml" decode: num-worker: 1 tp: 4 @@ -697,7 +697,7 @@ minimaxm3-fp8-gb300-dynamo-vllm-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c1-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp8-eagle3-c1-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -711,7 +711,7 @@ minimaxm3-fp8-gb300-dynamo-vllm-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c4-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp8-eagle3-c4-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -725,7 +725,7 @@ minimaxm3-fp8-gb300-dynamo-vllm-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tp8-eagle3-c8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -739,7 +739,7 @@ minimaxm3-fp8-gb300-dynamo-vllm-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-dep8-eagle3-c64-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-dep8-eagle3-c64-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -753,7 +753,7 @@ minimaxm3-fp8-gb300-dynamo-vllm-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/2p1d-dep2-dep8-eagle3-c512-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-2p1d-dep2-dep8-eagle3-c512-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -767,7 +767,7 @@ minimaxm3-fp8-gb300-dynamo-vllm-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/3p1d-dep2-dep8-eagle3-c256-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-3p1d-dep2-dep8-eagle3-c256-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -781,7 +781,7 @@ minimaxm3-fp8-gb300-dynamo-vllm-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c1024-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-4p1d-dep2-dep8-eagle3-c1024-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -795,7 +795,7 @@ minimaxm3-fp8-gb300-dynamo-vllm-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c2048-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-4p1d-dep2-dep8-eagle3-c2048-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -809,7 +809,7 @@ minimaxm3-fp8-gb300-dynamo-vllm-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/6p1d-dep2-dep8-eagle3-c2048-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-6p1d-dep2-dep8-eagle3-c2048-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -841,7 +841,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p1d-dep2-dep8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-dep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -856,7 +856,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p1d-dep2-tep8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p1d-dep2-tep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -871,7 +871,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p2d-dep2-tep8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-1p2d-dep2-tep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -886,7 +886,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p1d-dep2-dep8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-2p1d-dep2-dep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -901,7 +901,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p2d-dep2-tep8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-2p2d-dep2-tep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -916,7 +916,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p4d-dep2-tep4-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-2p4d-dep2-tep4-stp.yaml" decode: num-worker: 4 tp: 4 @@ -931,7 +931,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/3p1d-dep2-dep8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-3p1d-dep2-dep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -946,7 +946,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/3p1d-dep2-dep16-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-3p1d-dep2-dep16-stp.yaml" decode: num-worker: 1 tp: 16 @@ -961,7 +961,7 @@ minimaxm3-fp8-gb300-dynamo-vllm: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/6p1d-dep2-dep8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp8/8k1k/disagg-6p1d-dep2-dep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -995,7 +995,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p1d-dep4-dep16-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-1p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 16 @@ -1010,7 +1010,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-tep4-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-1p2d-dep4-tep4-stp.yaml" decode: num-worker: 2 tp: 4 @@ -1025,7 +1025,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-tep8-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-1p2d-dep4-tep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -1040,7 +1040,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/2p1d-dep4-dep16-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-2p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 16 @@ -1055,7 +1055,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/3p1d-dep4-dep16-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-3p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 16 @@ -1070,7 +1070,7 @@ minimaxm3-fp8-gb200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/5p1d-dep4-dep16-8k1k.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp8/8k1k/disagg-5p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 16 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index ca0a27caec..4d713b755e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -23,7 +23,7 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen1_dep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen1_dep8_batch8_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p1d-dep8-b8-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 8 @@ -38,7 +38,7 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen3_tep8_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen3_tep8_batch16_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp3.yaml" decode: num-worker: 3 tp: 8 @@ -53,7 +53,7 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen5_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen5_tep8_batch1_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b1-eplb0-mtp3.yaml" decode: num-worker: 5 tp: 8 @@ -68,7 +68,7 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen5_tep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen5_tep8_batch8_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b8-eplb0-mtp3.yaml" decode: num-worker: 5 tp: 8 @@ -83,7 +83,7 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx3_gen1_dep8_batch64_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/mtp/ctx3_gen1_dep8_batch64_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 8 @@ -98,7 +98,7 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx5_gen1_dep8_batch192_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/mtp/ctx5_gen1_dep8_batch192_eplb0_mtp1.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-5p1d-dep8-b192-eplb0-mtp1.yaml" decode: num-worker: 1 tp: 8 @@ -113,7 +113,7 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx5_gen2_dep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/mtp/ctx5_gen2_dep8_batch32_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-5p2d-dep8-b32-eplb0-mtp3.yaml" decode: num-worker: 2 tp: 8 @@ -129,7 +129,7 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx1_gen5_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/stp/ctx1_gen5_tep8_batch1_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b1-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 8 @@ -143,7 +143,7 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx1_gen5_tep8_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/stp/ctx1_gen5_tep8_batch8_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-1p5d-tep8-b16-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 8 @@ -157,7 +157,7 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx2_gen5_tep8_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/stp/ctx2_gen5_tep8_batch64_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-2p5d-tep8-b64-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 8 @@ -171,7 +171,7 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx4_gen1_dep8_batch192_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/stp/ctx4_gen1_dep8_batch192_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-4p1d-dep8-b192-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 8 @@ -185,7 +185,7 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx4_gen3_dep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/stp/ctx4_gen3_dep8_batch32_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-4p3d-dep8-b32-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 8 @@ -199,7 +199,7 @@ dsr1-fp4-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx7_gen2_dep8_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/stp/ctx7_gen2_dep8_batch128_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp4/8k1k/disagg-7p2d-dep8-b128-eplb0-mtp0.yaml" decode: num-worker: 2 tp: 8 @@ -232,7 +232,7 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen6_tp8_batch8_eplb0_mtp3_8.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen6_tp8_batch8_eplb0_mtp3_8.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p6d-tp8-b1-eplb0-mtp3-c8.yaml" decode: num-worker: 6 tp: 8 @@ -247,7 +247,7 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen2_tp8_batch32_eplb0_mtp3_8.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen2_tp8_batch32_eplb0_mtp3_8.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p2d-tp8-b4-eplb0-mtp3-c8.yaml" decode: num-worker: 2 tp: 8 @@ -262,7 +262,7 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen6_tp8_batch8_eplb0_mtp3_48.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen6_tp8_batch8_eplb0_mtp3_48.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p6d-tp8-b8-eplb0-mtp3-c48.yaml" decode: num-worker: 6 tp: 8 @@ -277,7 +277,7 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen4_tp8_batch16_eplb0_mtp3_64.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen4_tp8_batch16_eplb0_mtp3_64.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p4d-tp8-b16-eplb0-mtp3-c64.yaml" decode: num-worker: 4 tp: 8 @@ -293,7 +293,7 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx2_gen3_dep8_batch8_eplb0_mtp3_224.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/mtp/ctx2_gen3_dep8_batch8_eplb0_mtp3_224.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-2p3d-dep8-b8-eplb0-mtp3-c224.yaml" decode: num-worker: 3 tp: 8 @@ -308,7 +308,7 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx2_gen1_dep8_batch32_eplb0_mtp3_288.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/mtp/ctx2_gen1_dep8_batch32_eplb0_mtp3_288.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-2p1d-dep8-b32-eplb0-mtp3-c288.yaml" decode: num-worker: 1 tp: 8 @@ -323,7 +323,7 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx4_gen1_dep8_batch128_eplb0_mtp2_1088.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/mtp/ctx4_gen1_dep8_batch128_eplb0_mtp2_1088.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-4p1d-dep8-b128-eplb0-mtp2-c1088.yaml" decode: num-worker: 1 tp: 8 @@ -339,7 +339,7 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_tp8_batch1_eplb0_mtp0_1.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_tp8_batch1_eplb0_mtp0_1.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p1d-tp8-b1-eplb0-mtp0-c1.yaml" decode: num-worker: 1 tp: 8 @@ -353,7 +353,7 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen4_tp8_batch32_eplb0_mtp0_32.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen4_tp8_batch32_eplb0_mtp0_32.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p4d-tp8-b8-eplb0-mtp0-c32.yaml" decode: num-worker: 4 tp: 8 @@ -367,7 +367,7 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen4_tp8_batch32_eplb0_mtp0_128.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen4_tp8_batch32_eplb0_mtp0_128.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p4d-tp8-b32-eplb0-mtp0-c128.yaml" decode: num-worker: 4 tp: 8 @@ -381,7 +381,7 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen6_tp8_batch16_eplb0_mtp0_96.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen6_tp8_batch16_eplb0_mtp0_96.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p6d-tp8-b16-eplb0-mtp0-c96.yaml" decode: num-worker: 6 tp: 8 @@ -396,7 +396,7 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_dep8_batch128_eplb0_mtp0_128.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_dep8_batch128_eplb0_mtp0_128.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p1d-dep8-b128-eplb0-mtp0-c128.yaml" decode: num-worker: 1 tp: 8 @@ -410,7 +410,7 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen2_dep8_batch64_eplb0_mtp0_128.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen2_dep8_batch64_eplb0_mtp0_128.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p2d-dep8-b64-eplb0-mtp0-c128.yaml" decode: num-worker: 2 tp: 8 @@ -424,7 +424,7 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_dep8_batch256_eplb0_mtp0_256.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_dep8_batch256_eplb0_mtp0_256.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-1p1d-dep8-b256-eplb0-mtp0-c256.yaml" decode: num-worker: 1 tp: 8 @@ -438,7 +438,7 @@ dsr1-fp8-b200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx2_gen1_dep8_batch640_eplb0_mtp0_640.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/stp/ctx2_gen1_dep8_batch640_eplb0_mtp0_640.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b200-fp8/8k1k/disagg-2p1d-dep8-b640-eplb0-mtp0-c640.yaml" decode: num-worker: 1 tp: 8 @@ -470,7 +470,7 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx10_gen1_dep8_batch256_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/mtp/ctx10_gen1_dep8_batch256_eplb0_mtp1.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp1.yaml" decode: num-worker: 1 tp: 8 @@ -485,7 +485,7 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep4_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep4_batch8_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep4-b8-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 4 @@ -500,7 +500,7 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -515,7 +515,7 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -530,7 +530,7 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx3_gen1_dep8_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/mtp/ctx3_gen1_dep8_batch16_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-3p1d-dep8-b16-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 8 @@ -545,7 +545,7 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx9_gen1_dep8_batch128_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/mtp/ctx9_gen1_dep8_batch128_eplb0_mtp1.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-9p1d-dep8-b128-eplb0-mtp1.yaml" decode: num-worker: 1 tp: 8 @@ -561,7 +561,7 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep4_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep4_batch32_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p3d-tep4-b32-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 4 @@ -575,7 +575,7 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 8 @@ -589,7 +589,7 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p3d-tep8-b1-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 8 @@ -603,7 +603,7 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen4_tep4_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen4_tep4_batch2_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-1p4d-tep4-b2-eplb0-mtp0.yaml" decode: num-worker: 4 tp: 4 @@ -617,7 +617,7 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx5_gen2_dep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/stp/ctx5_gen2_dep8_batch32_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-5p2d-dep8-b32-eplb0-mtp0.yaml" decode: num-worker: 2 tp: 8 @@ -631,7 +631,7 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx6_gen1_dep8_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/stp/ctx6_gen1_dep8_batch128_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-6p1d-dep8-b128-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 8 @@ -645,7 +645,7 @@ dsr1-fp4-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx8_gen1_dep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/stp/ctx8_gen1_dep8_batch256_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp4/8k1k/disagg-8p1d-dep8-b256-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 8 @@ -676,7 +676,7 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen2_tp8_batch16_eplb0_mtp3_40.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen2_tp8_batch16_eplb0_mtp3_40.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p2d-tp8-b16-eplb0-mtp3-c40.yaml" decode: num-worker: 2 tp: 8 @@ -691,7 +691,7 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen4_tp8_batch1_eplb0_mtp3_8.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen4_tp8_batch1_eplb0_mtp3_8.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p4d-tp8-b1-eplb0-mtp3-c8.yaml" decode: num-worker: 4 tp: 8 @@ -706,7 +706,7 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen4_tp8_batch4_eplb0_mtp3_20.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen4_tp8_batch4_eplb0_mtp3_20.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p4d-tp8-b4-eplb0-mtp3-c20.yaml" decode: num-worker: 4 tp: 8 @@ -721,7 +721,7 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen1_dp8_batch8_eplb0_mtp3_72.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen1_dp8_batch8_eplb0_mtp3_72.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p1d-dp8-b8-eplb0-mtp3-c72.yaml" decode: num-worker: 1 tp: 8 @@ -736,7 +736,7 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx2_gen1_dp8_batch16_eplb0_mtp3_144.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/mtp/ctx2_gen1_dp8_batch16_eplb0_mtp3_144.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-2p1d-dp8-b16-eplb0-mtp3-c144.yaml" decode: num-worker: 1 tp: 8 @@ -751,7 +751,7 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx4_gen1_dp8_batch64_eplb0_mtp2_512.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/mtp/ctx4_gen1_dp8_batch64_eplb0_mtp2_512.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-4p1d-dp8-b64-eplb0-mtp2-c512.yaml" decode: num-worker: 1 tp: 8 @@ -769,7 +769,7 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx1_gen4_tp8_batch16_eplb0_mtp0_64.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/stp/ctx1_gen4_tp8_batch16_eplb0_mtp0_64.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p4d-tp8-b16-eplb0-mtp0-c64.yaml" decode: num-worker: 4 tp: 8 @@ -783,7 +783,7 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx1_gen8_tp8_batch2_eplb0_mtp0_16.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/stp/ctx1_gen8_tp8_batch2_eplb0_mtp0_16.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-1p8d-tp8-b1-eplb0-mtp0-c16.yaml" decode: num-worker: 8 tp: 8 @@ -797,7 +797,7 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx2_gen1_dp8_batch32_eplb0_mtp0_256.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/stp/ctx2_gen1_dp8_batch32_eplb0_mtp0_256.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-2p1d-dp8-b32-eplb0-mtp0-c256.yaml" decode: num-worker: 1 tp: 8 @@ -811,7 +811,7 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx3_gen1_dp8_batch64_eplb0_mtp0_512.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/stp/ctx3_gen1_dp8_batch64_eplb0_mtp0_512.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-3p1d-dp8-b64-eplb0-mtp0-c512.yaml" decode: num-worker: 1 tp: 8 @@ -825,7 +825,7 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx3_gen5_tp8_batch64_eplb0_mtp0_256.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/stp/ctx3_gen5_tp8_batch64_eplb0_mtp0_256.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-3p5d-tp8-b64-eplb0-mtp0-c256.yaml" decode: num-worker: 5 tp: 8 @@ -839,7 +839,7 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx5_gen1_dp8_batch128_eplb0_mtp0_1075.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/stp/ctx5_gen1_dp8_batch128_eplb0_mtp0_1075.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-5p1d-dp8-b128-eplb0-mtp0-c1075.yaml" decode: num-worker: 1 tp: 8 @@ -853,7 +853,7 @@ dsr1-fp8-b300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx7_gen1_dep8_batch384_eplb0_mtp0_3072.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/stp/ctx7_gen1_dep8_batch384_eplb0_mtp0_3072.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/b300-fp8/8k1k/disagg-7p1d-dep8-b384-eplb0-mtp0-c3072.yaml" decode: num-worker: 1 tp: 8 @@ -1416,7 +1416,7 @@ kimik3-fp4-h200-vllm-agentic-latency: ep: 32 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-h200-tp16dp2ep32-latency-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/h200-fp4/agentx/agg-tp16dp2ep32-latency.yaml" kimik3-fp4-h200-vllm-agentic-balanced: image: vllm/vllm-openai:kimi-k3 @@ -1442,7 +1442,7 @@ kimik3-fp4-h200-vllm-agentic-balanced: ep: 32 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-balanced-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml" kimik3-fp4-h200-vllm-agentic-simple: image: vllm/vllm-openai:kimi-k3 @@ -1469,7 +1469,7 @@ kimik3-fp4-h200-vllm-agentic-simple: ep: 32 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-vllm-simple-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml" # NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html # does not have a B300-specific recipe, so this config reuses the existing # Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. @@ -1667,7 +1667,7 @@ dsv4-fp8-h200-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-h200-tp8-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml" # MTP variant of dsv4-fp8-h200-sglang. Mirrors the non-MTP recipe (same image, # runner pool, search space) and adds EAGLE speculative decoding via # --speculative-algorithm EAGLE with the (3,1,4) chain matching dsv4-fp4-b300-sglang-mtp. @@ -1895,7 +1895,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b1-eplb0-mtp3-c9.yaml" decode: num-worker: 7 tp: 8 @@ -1910,7 +1910,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b32-eplb0-mtp3-c28.yaml" decode: num-worker: 7 tp: 8 @@ -1925,7 +1925,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c8_ctx1_gen6_tep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c8_ctx1_gen6_tep8_batch32_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p6d-tep8-b32-eplb0-mtp3-c48.yaml" decode: num-worker: 6 tp: 8 @@ -1940,7 +1940,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp2.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p3d-tep8-b32-eplb0-mtp2-c48.yaml" decode: num-worker: 3 tp: 8 @@ -1955,7 +1955,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c32_ctx3_gen5_tep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c32_ctx3_gen5_tep8_batch32_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p5d-tep8-b32-eplb0-mtp3-c160.yaml" decode: num-worker: 5 tp: 8 @@ -1970,7 +1970,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c64_ctx1_gen1_dep8_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c64_ctx1_gen1_dep8_batch32_eplb0_mtp2.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p1d-dep8-b32-eplb0-mtp2-c64.yaml" decode: num-worker: 1 tp: 8 @@ -1985,7 +1985,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c128_ctx2_gen1_dep8_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c128_ctx2_gen1_dep8_batch32_eplb0_mtp2.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-2p1d-dep8-b32-eplb0-mtp2-c128.yaml" decode: num-worker: 1 tp: 8 @@ -2000,7 +2000,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c256_ctx3_gen1_dep8_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c256_ctx3_gen1_dep8_batch32_eplb0_mtp2.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p1d-dep8-b32-eplb0-mtp2-c256.yaml" decode: num-worker: 1 tp: 8 @@ -2015,7 +2015,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c512_ctx3_gen1_dep8_batch64_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c512_ctx3_gen1_dep8_batch64_eplb0_mtp1.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp1-c512.yaml" decode: num-worker: 1 tp: 8 @@ -2030,7 +2030,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b1-eplb0-mtp0-c9.yaml" decode: num-worker: 7 tp: 8 @@ -2044,7 +2044,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p7d-tep8-b32-eplb0-mtp0-c28.yaml" decode: num-worker: 7 tp: 8 @@ -2058,7 +2058,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c8_ctx1_gen6_tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c8_ctx1_gen6_tep8_batch16_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p6d-tep8-b16-eplb0-mtp0-c48.yaml" decode: num-worker: 6 tp: 8 @@ -2072,7 +2072,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p3d-tep8-b32-eplb0-mtp0-c48.yaml" decode: num-worker: 3 tp: 8 @@ -2086,7 +2086,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c32_ctx2_gen5_tep8_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c32_ctx2_gen5_tep8_batch128_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-2p5d-tep8-b128-eplb0-mtp0-c160.yaml" decode: num-worker: 5 tp: 8 @@ -2100,7 +2100,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c64_ctx2_gen3_dep8_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c64_ctx2_gen3_dep8_batch128_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-2p3d-dep8-b128-eplb0-mtp0-c192.yaml" decode: num-worker: 3 tp: 8 @@ -2114,7 +2114,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c128_ctx1_gen1_dep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c128_ctx1_gen1_dep8_batch256_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-1p1d-dep8-b256-eplb0-mtp0-c128.yaml" decode: num-worker: 1 tp: 8 @@ -2128,7 +2128,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c256_ctx5_gen3_dep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c256_ctx5_gen3_dep8_batch256_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-5p3d-dep8-b256-eplb0-mtp0-c768.yaml" decode: num-worker: 3 tp: 8 @@ -2142,7 +2142,7 @@ dsr1-fp8-h200-dynamo-trt: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c512_ctx3_gen1_dep8_batch512_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c512_ctx3_gen1_dep8_batch512_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h200-fp8/8k1k/disagg-3p1d-dep8-b512-eplb0-mtp0-c512.yaml" decode: num-worker: 1 tp: 8 @@ -2175,7 +2175,7 @@ dsr1-fp8-h100-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch1_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b1-eplb0-mtp3.yaml" decode: num-worker: 3 tp: 16 @@ -2190,7 +2190,7 @@ dsr1-fp8-h100-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch2_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b2-eplb0-mtp3.yaml" decode: num-worker: 3 tp: 16 @@ -2205,7 +2205,7 @@ dsr1-fp8-h100-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch8_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b8-eplb0-mtp3.yaml" decode: num-worker: 3 tp: 16 @@ -2220,7 +2220,7 @@ dsr1-fp8-h100-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen1_dep16_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen1_dep16_batch4_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p1d-dep16-b4-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 16 @@ -2237,7 +2237,7 @@ dsr1-fp8-h100-dynamo-trt: # dp-attn: true # additional-settings: # # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen2_tep16_batch32_eplb0_mtp3.yaml - # - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen2_tep16_batch32_eplb0_mtp3.yaml" + # - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p2d-tep16-b32-eplb0-mtp3.yaml" # decode: # num-worker: 2 # tp: 16 @@ -2252,7 +2252,7 @@ dsr1-fp8-h100-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx2_gen1_dep16_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx2_gen1_dep16_batch8_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-2p1d-dep16-b8-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 16 @@ -2267,7 +2267,7 @@ dsr1-fp8-h100-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch1_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b1-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 16 @@ -2281,7 +2281,7 @@ dsr1-fp8-h100-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch2_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b2-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 16 @@ -2295,7 +2295,7 @@ dsr1-fp8-h100-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch8_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p3d-tep16-b8-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 16 @@ -2309,7 +2309,7 @@ dsr1-fp8-h100-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen2_tep16_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen2_tep16_batch64_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-1p2d-tep16-b64-eplb0-mtp0.yaml" decode: num-worker: 2 tp: 16 @@ -2323,7 +2323,7 @@ dsr1-fp8-h100-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx2_gen1_dep16_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx2_gen1_dep16_batch16_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/h100-fp8/8k1k/disagg-2p1d-dep16-b16-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 16 @@ -2354,7 +2354,7 @@ dsr1-fp8-h100-dynamo-sglang: # ep: 1 # dp-attn: false # additional-settings: - # - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-tp.yaml" + # - "CONFIG_FILE=recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-b128-c1x2x4x8x16x32x64x128-stp.yaml" # decode: # num-worker: 1 # tp: 16 @@ -2368,7 +2368,7 @@ dsr1-fp8-h100-dynamo-sglang: # ep: 1 # dp-attn: false # additional-settings: - # - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-dep.yaml" + # - "CONFIG_FILE=recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-ep16-dp16-b64-c1x2x4x8x16x32x64-stp.yaml" # decode: # num-worker: 1 # tp: 16 @@ -2383,7 +2383,7 @@ dsr1-fp8-h100-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h100/8k1k/mtp/h100-fp8-1p1d-max-tp-mtp.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-b128-c1x2x4x8x16x32x64x128-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -2398,7 +2398,7 @@ dsr1-fp8-h100-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h100/8k1k/mtp/h100-fp8-1p1d-max-dep-mtp.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/h100-fp8/8k1k/disagg-1p1d-p-tp16-d-tp16-ep16-dp16-b64-c1x2x4x8x16x32x64-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -2431,7 +2431,7 @@ dsr1-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx1_gen4_tep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx1_gen4_tep8_batch8_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -2446,7 +2446,7 @@ dsr1-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx3_gen1_dep32_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx3_gen1_dep32_batch4_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-3p1d-dep32-b4-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -2461,7 +2461,7 @@ dsr1-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx7_gen1_dep16_batch64_eplb256_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx7_gen1_dep16_batch64_eplb256_mtp1.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep16-b64-eplb256-mtp1.yaml" decode: num-worker: 1 tp: 16 @@ -2476,7 +2476,7 @@ dsr1-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx8_gen1_dep32_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx8_gen1_dep32_batch16_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep32-b16-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -2491,7 +2491,7 @@ dsr1-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx11_gen1_dep16_batch256_eplb256_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx11_gen1_dep16_batch256_eplb256_mtp1.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-11p1d-dep16-b256-eplb256-mtp1.yaml" decode: num-worker: 1 tp: 16 @@ -2507,7 +2507,7 @@ dsr1-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch16_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b16-eplb0-mtp0.yaml" decode: num-worker: 4 tp: 8 @@ -2521,7 +2521,7 @@ dsr1-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml" decode: num-worker: 4 tp: 8 @@ -2535,7 +2535,7 @@ dsr1-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 32 @@ -2549,7 +2549,7 @@ dsr1-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx7_gen1_dep32_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx7_gen1_dep32_batch32_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep32-b32-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 32 @@ -2563,7 +2563,7 @@ dsr1-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx8_gen1_dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx8_gen1_dep16_batch128_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep16-b128-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 16 @@ -2577,7 +2577,7 @@ dsr1-fp4-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx10_gen1_dep16_batch256_eplb256_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx10_gen1_dep16_batch256_eplb256_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-10p1d-dep16-b256-eplb256-mtp0.yaml" decode: num-worker: 1 tp: 16 @@ -2609,7 +2609,7 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep8_batch64_eplb0_mtp3_666.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep8_batch64_eplb0_mtp3_666.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp3-c666.yaml" decode: num-worker: 1 tp: 8 @@ -2624,7 +2624,7 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx5_gen1_dep16_batch32_eplb0_mtp3_666.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx5_gen1_dep16_batch32_eplb0_mtp3_666.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b32-eplb0-mtp3-c666.yaml" decode: num-worker: 1 tp: 16 @@ -2639,7 +2639,7 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep16_batch16_eplb0_mtp3_333.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep16_batch16_eplb0_mtp3_333.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b16-eplb0-mtp3-c333.yaml" decode: num-worker: 1 tp: 16 @@ -2654,7 +2654,7 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx4_gen1_dep32_batch8_eplb0_mtp3_333.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx4_gen1_dep32_batch8_eplb0_mtp3_333.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b8-eplb0-mtp3-c333.yaml" decode: num-worker: 1 tp: 32 @@ -2669,7 +2669,7 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx2_gen1_dep32_batch2_eplb0_mtp3_90.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx2_gen1_dep32_batch2_eplb0_mtp3_90.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b2-eplb0-mtp3-c90.yaml" decode: num-worker: 1 tp: 32 @@ -2684,7 +2684,7 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch4_eplb0_mtp3_15.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch4_eplb0_mtp3_15.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp3-c15.yaml" decode: num-worker: 3 tp: 8 @@ -2699,7 +2699,7 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch2_eplb0_mtp3_6.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch2_eplb0_mtp3_6.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b2-eplb0-mtp3-c6.yaml" decode: num-worker: 3 tp: 8 @@ -2714,7 +2714,7 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx5_gen1_dep16_batch64_eplb0_mtp0_1229.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx5_gen1_dep16_batch64_eplb0_mtp0_1229.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b64-eplb0-mtp0-c1229.yaml" decode: num-worker: 1 tp: 16 @@ -2728,7 +2728,7 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx4_gen1_dep32_batch16_eplb0_mtp0_666.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx4_gen1_dep32_batch16_eplb0_mtp0_666.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b16-eplb0-mtp0-c666.yaml" decode: num-worker: 1 tp: 32 @@ -2742,7 +2742,7 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx3_gen1_dep16_batch32_eplb0_mtp0_615.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx3_gen1_dep16_batch32_eplb0_mtp0_615.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b32-eplb0-mtp0-c615.yaml" decode: num-worker: 1 tp: 16 @@ -2756,7 +2756,7 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0_333.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0_333.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0-c333.yaml" decode: num-worker: 1 tp: 32 @@ -2770,7 +2770,7 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0_63.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0_63.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0-c63.yaml" decode: num-worker: 3 tp: 8 @@ -2784,7 +2784,7 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch4_eplb0_mtp0_18.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch4_eplb0_mtp0_18.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp0-c18.yaml" decode: num-worker: 3 tp: 8 @@ -2798,7 +2798,7 @@ dsr1-fp8-gb200-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0_6.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0_6.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b1-eplb0-mtp0-c6.yaml" decode: num-worker: 3 tp: 8 @@ -2830,7 +2830,7 @@ dsr1-fp8-gb200-dynamo-sglang: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/low-latency.yaml - - "CONFIG_FILE=recipes/gb200-fp8/8k1k/low-latency.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/gb200-fp8/8k1k/disagg-tp8-stp-variants.yaml" decode: num-worker: 1 tp: 8 @@ -2846,7 +2846,7 @@ dsr1-fp8-gb200-dynamo-sglang: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/mid-curve.yaml - - "CONFIG_FILE=recipes/gb200-fp8/8k1k/mid-curve.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/gb200-fp8/8k1k/disagg-5p1d-p-tp8-ep8-dp8-d-tp32-ep32-dp32-b8192-c512x1024x2048x6144-stp.yaml" decode: num-worker: 1 tp: 32 @@ -2862,7 +2862,7 @@ dsr1-fp8-gb200-dynamo-sglang: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/max_tpt.yaml - - "CONFIG_FILE=recipes/gb200-fp8/8k1k/max_tpt.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/gb200-fp8/8k1k/disagg-6p1d-p-tp8-ep8-dp8-d-tp24-ep24-dp24-b8192-c2048x4096x6144-stp.yaml" decode: num-worker: 1 tp: 24 @@ -2894,7 +2894,7 @@ dsr1-fp8-gb300-dynamo-sglang: dp-attn: false additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/low-latency.yaml - - "CONFIG_FILE=recipes/gb300-fp8/8k1k/stp/low-latency.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c4x8-stp.yaml" decode: num-worker: 1 tp: 4 @@ -2910,7 +2910,7 @@ dsr1-fp8-gb300-dynamo-sglang: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/mid.yaml - - "CONFIG_FILE=recipes/gb300-fp8/8k1k/stp/mid.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/gb300-fp8/8k1k/disagg-5p1d-p-tp8-ep8-dp8-d-tp32-ep32-dp32-b45000-c128x256x512x1024-stp.yaml" decode: num-worker: 1 tp: 32 @@ -2926,7 +2926,7 @@ dsr1-fp8-gb300-dynamo-sglang: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/max.yaml - - "CONFIG_FILE=recipes/gb300-fp8/8k1k/stp/max.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/gb300-fp8/8k1k/disagg-6p1d-p-tp8-ep8-dp8-d-tp24-ep24-dp24-b45000-c2048x4096-stp.yaml" decode: num-worker: 1 tp: 24 @@ -2958,7 +2958,7 @@ dsr1-fp4-gb200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/gb200-fp4/8k1k/low-latency.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/gb200-fp4/8k1k/disagg-1p4d-p-tp4-d-tp4-c4x8-stp.yaml" decode: num-worker: 4 tp: 4 @@ -2974,7 +2974,7 @@ dsr1-fp4-gb200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/gb200-fp4/8k1k/mid-curve.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/gb200-fp4/8k1k/disagg-6p1d-p-tp4-d-tp48-ep48-dp48-b16384-c512x2048x4096-stp.yaml" decode: num-worker: 1 tp: 48 @@ -2990,7 +2990,7 @@ dsr1-fp4-gb200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/gb200-fp4/8k1k/max-tpt.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/gb200-fp4/8k1k/disagg-10p1d-p-tp4-d-tp32-ep32-dp32-b16384-c2048-stp.yaml" decode: num-worker: 1 tp: 32 @@ -3023,7 +3023,7 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen3_tep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen3_tep8_batch8_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b8-eplb0-mtp3.yaml" decode: num-worker: 3 tp: 8 @@ -3038,7 +3038,7 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -3053,7 +3053,7 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -3068,7 +3068,7 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx4_gen1_dep32_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx4_gen1_dep32_batch4_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep32-b4-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -3083,7 +3083,7 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx8_gen1_dep32_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx8_gen1_dep32_batch8_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-8p1d-dep32-b8-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -3098,7 +3098,7 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep8_batch256_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep8_batch256_eplb0_mtp1.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp1.yaml" decode: num-worker: 1 tp: 8 @@ -3113,7 +3113,7 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep16_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep16_batch32_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep16-b32-eplb0-mtp3.yaml" decode: num-worker: 1 tp: 16 @@ -3128,7 +3128,7 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx13_gen1_dep16_batch64_eplb256_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx13_gen1_dep16_batch64_eplb256_mtp3.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-13p1d-dep16-b64-eplb256-mtp3.yaml" decode: num-worker: 1 tp: 16 @@ -3143,7 +3143,7 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 8 @@ -3157,7 +3157,7 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml" decode: num-worker: 4 tp: 8 @@ -3171,7 +3171,7 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch2_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b2-eplb0-mtp0.yaml" decode: num-worker: 4 tp: 8 @@ -3185,7 +3185,7 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen5_tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen5_tep4_batch4_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p5d-tep4-b4-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 4 @@ -3199,7 +3199,7 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx7_gen1_dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx7_gen1_dep32_batch16_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep32-b16-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 32 @@ -3213,7 +3213,7 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx9_gen1_dep16_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx9_gen1_dep16_batch64_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep16-b64-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 16 @@ -3227,7 +3227,7 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx11_gen3_dep4_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx11_gen3_dep4_batch256_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-11p3d-dep4-b256-eplb0-mtp0.yaml" decode: num-worker: 3 tp: 4 @@ -3241,7 +3241,7 @@ dsr1-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx14_gen1_dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx14_gen1_dep16_batch128_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-14p1d-dep16-b128-eplb0-mtp0.yaml" decode: num-worker: 1 tp: 16 @@ -3272,7 +3272,7 @@ dsr1-fp4-gb300-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/8k1k/low_latency.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/gb300-fp4/8k1k/disagg-1p4d-p-tp4-d-tp4-c4x8x32x64-stp.yaml" decode: num-worker: 4 tp: 4 @@ -3288,7 +3288,7 @@ dsr1-fp4-gb300-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/8k1k/mid_curve.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/gb300-fp4/8k1k/disagg-6p1d-p-tp4-d-tp48-ep48-dp48-b16384-c512x2048x4096-stp.yaml" decode: num-worker: 1 tp: 48 @@ -3304,7 +3304,7 @@ dsr1-fp4-gb300-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/8k1k/max_tpt.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/gb300-fp4/8k1k/disagg-10p1d-p-tp4-d-tp32-ep32-dp32-b16384-c2048-stp.yaml" decode: num-worker: 1 tp: 32 @@ -3337,7 +3337,7 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_8.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_8.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3-c8.yaml" decode: num-worker: 4 tp: 8 @@ -3352,7 +3352,7 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3_24.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3_24.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3-c24.yaml" decode: num-worker: 4 tp: 8 @@ -3367,7 +3367,7 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx6_gen1_dep32_batch8_eplb0_mtp3_333.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx6_gen1_dep32_batch8_eplb0_mtp3_333.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b8-eplb0-mtp3-c333.yaml" decode: num-worker: 1 tp: 32 @@ -3382,7 +3382,7 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx8_gen1_dep16_batch32_eplb0_mtp3_666.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx8_gen1_dep16_batch32_eplb0_mtp3_666.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-8p1d-dep16-b32-eplb0-mtp3-c666.yaml" decode: num-worker: 1 tp: 16 @@ -3397,7 +3397,7 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx10_gen1_dep16_batch64_eplb0_mtp1_1229.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx10_gen1_dep16_batch64_eplb0_mtp1_1229.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-10p1d-dep16-b64-eplb0-mtp1-c1229.yaml" decode: num-worker: 1 tp: 16 @@ -3412,7 +3412,7 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx7_gen1_dep8_batch128_eplb0_mtp1_1229.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx7_gen1_dep8_batch128_eplb0_mtp1_1229.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b128-eplb0-mtp1-c1229.yaml" decode: num-worker: 1 tp: 8 @@ -3427,7 +3427,7 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_4.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_4.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0-c4.yaml" decode: num-worker: 4 tp: 8 @@ -3441,7 +3441,7 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml" decode: num-worker: 4 tp: 8 @@ -3455,7 +3455,7 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch8_eplb0_mtp0_36.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch8_eplb0_mtp0_36.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp0-c36.yaml" decode: num-worker: 4 tp: 8 @@ -3469,7 +3469,7 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx6_gen1_dep32_batch16_eplb0_mtp0_512.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx6_gen1_dep32_batch16_eplb0_mtp0_512.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b16-eplb0-mtp0-c512.yaml" decode: num-worker: 1 tp: 32 @@ -3483,7 +3483,7 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx4_gen1_dep16_batch32_eplb0_mtp0_666.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx4_gen1_dep16_batch32_eplb0_mtp0_666.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-4p1d-dep16-b32-eplb0-mtp0-c666.yaml" decode: num-worker: 1 tp: 16 @@ -3497,7 +3497,7 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep16_batch64_eplb0_mtp0_1229.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep16_batch64_eplb0_mtp0_1229.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep16-b64-eplb0-mtp0-c1229.yaml" decode: num-worker: 1 tp: 16 @@ -3511,7 +3511,7 @@ dsr1-fp8-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep8_batch256_eplb0_mtp0_2151.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep8_batch256_eplb0_mtp0_2151.yaml" + - "CONFIG_FILE=recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b256-eplb0-mtp0-c2151.yaml" decode: num-worker: 1 tp: 8 @@ -3542,7 +3542,7 @@ dsr1-fp8-h200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs4-1p7d.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs4-1p7d-stp.yaml" decode: num-worker: 7 tp: 8 @@ -3557,7 +3557,7 @@ dsr1-fp8-h200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs8-1p6d.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs8-1p6d-stp.yaml" decode: num-worker: 6 tp: 8 @@ -3572,7 +3572,7 @@ dsr1-fp8-h200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs16-1p3d.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs16-1p3d-stp.yaml" decode: num-worker: 3 tp: 8 @@ -3587,7 +3587,7 @@ dsr1-fp8-h200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs64-2p3d.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs64-2p3d-stp.yaml" decode: num-worker: 3 tp: 8 @@ -3602,7 +3602,7 @@ dsr1-fp8-h200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs128-1p1d-dep.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs128-1p1d-dep-stp.yaml" decode: num-worker: 1 tp: 8 @@ -3617,7 +3617,7 @@ dsr1-fp8-h200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs4-1p7d-mtp.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs4-1p7d-mtp.yaml" decode: num-worker: 7 tp: 8 @@ -3632,7 +3632,7 @@ dsr1-fp8-h200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs8-1p6d-mtp.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs8-1p6d-mtp.yaml" decode: num-worker: 6 tp: 8 @@ -3647,7 +3647,7 @@ dsr1-fp8-h200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs16-1p3d-mtp.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs16-1p3d-mtp.yaml" decode: num-worker: 3 tp: 8 @@ -3662,7 +3662,7 @@ dsr1-fp8-h200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs64-2p3d-mtp.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs64-2p3d-mtp.yaml" decode: num-worker: 3 tp: 8 @@ -3677,7 +3677,7 @@ dsr1-fp8-h200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs128-1p1d-dep-mtp.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/h200-fp8/8k1k/disagg-bs128-1p1d-dep-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -3707,7 +3707,7 @@ dsr1-fp4-b200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:zip_override_stp_lowlat[0]" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml:zip_override_stp_lowlat[0]" decode: num-worker: 1 tp: 8 @@ -3720,7 +3720,7 @@ dsr1-fp4-b200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:zip_override_stp_lowlat[1]" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml:zip_override_stp_lowlat[1]" decode: num-worker: 5 tp: 8 @@ -3733,7 +3733,7 @@ dsr1-fp4-b200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:zip_override_stp_lowlat[2]" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml:zip_override_stp_lowlat[2]" decode: num-worker: 5 tp: 8 @@ -3746,7 +3746,7 @@ dsr1-fp4-b200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:override_stp_tp4" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml:override_stp_tp4" decode: num-worker: 1 tp: 8 @@ -3759,7 +3759,7 @@ dsr1-fp4-b200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:override_stp_maxtpt_7p2d" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml:override_stp_maxtpt_7p2d" decode: num-worker: 2 tp: 8 @@ -3789,7 +3789,7 @@ dsr1-fp8-b200-dynamo-sglang: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_0.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_lowlat_0.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p3d-p-tp8-dp8-d-tp8-b32-c128-stp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 3 @@ -3804,7 +3804,7 @@ dsr1-fp8-b200-dynamo-sglang: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_1.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_lowlat_1.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p4d-p-tp8-dp8-d-tp8-b32-c128-stp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 4 @@ -3819,7 +3819,7 @@ dsr1-fp8-b200-dynamo-sglang: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_2.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_lowlat_2.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b22-c8x16x32x64x128-stp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 6 @@ -3835,7 +3835,7 @@ dsr1-fp8-b200-dynamo-sglang: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_0.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_0.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p2d-p-tp8-dp8-d-tp8-ep8-dp8-b128-c288-stp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 2 @@ -3850,7 +3850,7 @@ dsr1-fp8-b200-dynamo-sglang: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_1.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_1.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p1d-p-tp8-dp8-d-tp8-ep8-dp8-b256-c160x288-stp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 @@ -3865,7 +3865,7 @@ dsr1-fp8-b200-dynamo-sglang: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_2.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_2.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-2p1d-p-tp8-dp8-d-tp8-ep8-dp8-b512-c512-stp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 @@ -3880,7 +3880,7 @@ dsr1-fp8-b200-dynamo-sglang: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_3.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_3.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-3p1d-p-tp8-dp8-d-tp8-ep8-dp8-b1024-c1024-stp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 @@ -3913,7 +3913,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_0.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_lowlat_0.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p3d-p-tp8-dp8-d-tp8-b32-c128-mtp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 3 @@ -3929,7 +3929,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_1.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_lowlat_1.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p4d-p-tp8-dp8-d-tp8-b32-c128-mtp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 4 @@ -3945,7 +3945,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_2.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_lowlat_2.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p6d-p-tp8-dp8-d-tp8-b22-c8x16x32x64x128-mtp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 6 @@ -3962,7 +3962,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_0.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_0.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p2d-p-tp8-dp8-d-tp8-ep8-dp8-b128-c288-mtp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 2 @@ -3978,7 +3978,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_1.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_1.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-1p1d-p-tp8-dp8-d-tp8-ep8-dp8-b256-c160x288-mtp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 @@ -3994,7 +3994,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_2.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_2.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-2p1d-p-tp8-dp8-d-tp8-ep8-dp8-b512-c512-mtp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 @@ -4010,7 +4010,7 @@ dsr1-fp8-b200-dynamo-sglang-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_3.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_3.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp8/8k1k/disagg-3p1d-p-tp8-dp8-d-tp8-ep8-dp8-b1024-c1024-mtp.yaml" router: { name: dynamo-router, version: "0.9.1" } decode: num-worker: 1 @@ -4043,7 +4043,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p5d-p-tp4-d-tp8-b512-c4x8x16x32-mtp.yaml" decode: num-worker: 5 tp: 8 @@ -4058,7 +4058,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p3d-p-tp4-d-tp8-b512-c32x64-mtp.yaml" decode: num-worker: 3 tp: 8 @@ -4073,7 +4073,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp4-d-tp8-b512-c32-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -4089,7 +4089,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_1p1d.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c512-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -4104,7 +4104,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_2p1d.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-2p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c768-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -4119,7 +4119,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_3p1d.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c1024-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -4134,7 +4134,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_4p1d.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c512-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -4149,7 +4149,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_5p1d.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-5p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b2048-c2048-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -4164,7 +4164,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_4p1d_c2048.yaml" + - "CONFIG_FILE=recipes/dsr1/sglang/b200-fp4/8k1k/disagg-mtp-4p1d-c2048.yaml" decode: num-worker: 1 tp: 8 @@ -4194,7 +4194,7 @@ kimik2.6-fp4-b200-dynamo-vllm: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml" + - "CONFIG_FILE=recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p1d-dep8-tp8-c1-stp.yaml" decode: num-worker: 1 tp: 8 @@ -4207,7 +4207,7 @@ kimik2.6-fp4-b200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml" + - "CONFIG_FILE=recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p8d-dep4-tp4-c32-stp.yaml" decode: num-worker: 8 tp: 4 @@ -4220,7 +4220,7 @@ kimik2.6-fp4-b200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml" + - "CONFIG_FILE=recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p8d-dep4-tp4-c128-stp.yaml" decode: num-worker: 8 tp: 4 @@ -4233,7 +4233,7 @@ kimik2.6-fp4-b200-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml" + - "CONFIG_FILE=recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p4d-dep4-tp4-c512-stp.yaml" decode: num-worker: 4 tp: 4 @@ -4246,7 +4246,7 @@ kimik2.6-fp4-b200-dynamo-vllm: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml" + - "CONFIG_FILE=recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-1p1d-dep8-dep8-c2048-stp.yaml" decode: num-worker: 1 tp: 1 @@ -4259,7 +4259,7 @@ kimik2.6-fp4-b200-dynamo-vllm: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml" + - "CONFIG_FILE=recipes/kimik2.6/vllm/b200-fp4/8k1k/disagg-2p1d-dep8-dep8-c8192-stp.yaml" decode: num-worker: 1 tp: 1 @@ -4291,7 +4291,7 @@ dsv4-fp4-b200-dynamo-vllm: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c1.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c1-stp.yaml" decode: num-worker: 1 tp: 8 @@ -4304,7 +4304,7 @@ dsv4-fp4-b200-dynamo-vllm: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c32-c128.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c32x128-stp.yaml" decode: num-worker: 1 tp: 8 @@ -4317,7 +4317,7 @@ dsv4-fp4-b200-dynamo-vllm: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c64.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c64-stp.yaml" decode: num-worker: 1 tp: 8 @@ -4330,7 +4330,7 @@ dsv4-fp4-b200-dynamo-vllm: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c256.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c256-stp.yaml" decode: num-worker: 1 tp: 8 @@ -4343,7 +4343,7 @@ dsv4-fp4-b200-dynamo-vllm: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c512.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c512-stp.yaml" decode: num-worker: 1 tp: 8 @@ -4379,7 +4379,7 @@ dsv4-fp4-gb200-dynamo-vllm: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c1-stp.yaml" decode: num-worker: 1 tp: 8 @@ -4395,7 +4395,7 @@ dsv4-fp4-gb200-dynamo-vllm: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p4d-p-tp1-dp8-d-tp8-c256x512-stp.yaml" decode: num-worker: 4 tp: 8 @@ -4411,7 +4411,7 @@ dsv4-fp4-gb200-dynamo-vllm: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c256x512x1024-stp.yaml" decode: num-worker: 1 tp: 8 @@ -4427,7 +4427,7 @@ dsv4-fp4-gb200-dynamo-vllm: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-high-tpt-megamoe-stp.yaml" decode: num-worker: 1 tp: 8 @@ -4443,7 +4443,7 @@ dsv4-fp4-gb200-dynamo-vllm: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-3p1d-p-tp1-dp8-d-tp1-dp8-c4096-stp.yaml" decode: num-worker: 1 tp: 8 @@ -4561,7 +4561,7 @@ dsv4-fp4-gb200-dynamo-vllm-mtp2: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.27" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/agg-gb200-low-latency-mtp2.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/8k1k/agg-tp8-c1-mtp.yaml" decode: num-worker: 0 tp: 8 @@ -4579,7 +4579,7 @@ dsv4-fp4-gb200-dynamo-vllm-mtp2: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.27" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency-mtp2.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p4d-p-tp1-dp8-d-tp8-c16x32x64-mtp.yaml" decode: num-worker: 4 tp: 8 @@ -4598,7 +4598,7 @@ dsv4-fp4-gb200-dynamo-vllm-mtp2: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.27" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe-mtp2.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c128x256x512x1024-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -4617,7 +4617,7 @@ dsv4-fp4-gb200-dynamo-vllm-mtp2: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.27" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe-mtp2.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-high-tpt-megamoe-mtp2.yaml" decode: num-worker: 1 tp: 8 @@ -4652,7 +4652,7 @@ dsv4-fp4-gb200-dynamo-vllm-mtp2-nosynthetic: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/agg-gb200-low-latency-mtp2.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/8k1k/agg-tp8-c1-mtp.yaml" decode: num-worker: 0 tp: 8 @@ -4668,7 +4668,7 @@ dsv4-fp4-gb200-dynamo-vllm-mtp2-nosynthetic: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency-mtp2.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p4d-p-tp1-dp8-d-tp8-c16x32x64-mtp.yaml" decode: num-worker: 4 tp: 8 @@ -4685,7 +4685,7 @@ dsv4-fp4-gb200-dynamo-vllm-mtp2-nosynthetic: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe-mtp2.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c128x256x512x1024-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -4702,7 +4702,7 @@ dsv4-fp4-gb200-dynamo-vllm-mtp2-nosynthetic: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe-mtp2.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/8k1k/disagg-high-tpt-megamoe-mtp2.yaml" decode: num-worker: 1 tp: 8 @@ -4734,7 +4734,7 @@ dsv4-fp4-b200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p1d-tp8-tp8.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-tp8-tp8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -4748,7 +4748,7 @@ dsv4-fp4-b200-dynamo-sglang: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-dep8-tp8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -4762,7 +4762,7 @@ dsv4-fp4-b200-dynamo-sglang: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-dep8-dep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -4776,7 +4776,7 @@ dsv4-fp4-b200-dynamo-sglang: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p4d-dep8-tp8.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p4d-dep8-tp8-stp.yaml" decode: num-worker: 4 tp: 8 @@ -4790,7 +4790,7 @@ dsv4-fp4-b200-dynamo-sglang: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p2d-dep8-dep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -4804,7 +4804,7 @@ dsv4-fp4-b200-dynamo-sglang: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p4d-dep8-dep8.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p4d-dep8-dep8-stp.yaml" decode: num-worker: 4 tp: 8 @@ -4818,7 +4818,7 @@ dsv4-fp4-b200-dynamo-sglang: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-3p2d-dep8-dep8.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/8k1k/disagg-3p2d-dep8-dep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -4832,7 +4832,7 @@ dsv4-fp4-b200-dynamo-sglang: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-4p2d-dep8-dep8.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/8k1k/disagg-4p2d-dep8-dep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -4846,7 +4846,7 @@ dsv4-fp4-b200-dynamo-sglang: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-5p2d-dep8-dep8.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/8k1k/disagg-5p2d-dep8-dep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -4860,7 +4860,7 @@ dsv4-fp4-b200-dynamo-sglang: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-6p2d-dep8-dep8.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/8k1k/disagg-6p2d-dep8-dep8-stp.yaml" decode: num-worker: 2 tp: 8 @@ -4891,7 +4891,7 @@ dsv4-fp4-gb200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-tp8-tp8-4-c1.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p1d-tp8-tp8-4-c1-stp.yaml" decode: num-worker: 1 tp: 8 @@ -4905,7 +4905,7 @@ dsv4-fp4-gb200-dynamo-sglang: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p4d-dep8-tp8-10-c64.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p4d-dep8-tp8-10-c64-stp.yaml" decode: num-worker: 4 tp: 8 @@ -4919,7 +4919,7 @@ dsv4-fp4-gb200-dynamo-sglang: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p2d-dep8-dep16-10-c256.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p2d-dep8-dep16-10-c256-stp.yaml" decode: num-worker: 2 tp: 16 @@ -4933,7 +4933,7 @@ dsv4-fp4-gb200-dynamo-sglang: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-dep8-dep16-6-c512.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p1d-dep8-dep16-6-c512-stp.yaml" decode: num-worker: 1 tp: 16 @@ -4947,7 +4947,7 @@ dsv4-fp4-gb200-dynamo-sglang: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-2p1d-dep8-dep16-8-c1536.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-2p1d-dep8-dep16-8-c1536-stp.yaml" decode: num-worker: 1 tp: 16 @@ -4961,7 +4961,7 @@ dsv4-fp4-gb200-dynamo-sglang: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-4p1d-dep8-dep16-12-c4096.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-4p1d-dep8-dep16-12-c4096-stp.yaml" decode: num-worker: 1 tp: 16 @@ -4975,7 +4975,7 @@ dsv4-fp4-gb200-dynamo-sglang: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-5p1d-dep8-dep16-14-c8192.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-5p1d-dep8-dep16-14-c8192-stp.yaml" decode: num-worker: 1 tp: 16 @@ -4989,7 +4989,7 @@ dsv4-fp4-gb200-dynamo-sglang: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-6p1d-dep8-dep12-15-c8192.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-6p1d-dep8-dep12-15-c8192-stp.yaml" decode: num-worker: 1 tp: 12 @@ -5021,7 +5021,7 @@ qwen3.5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml" decode: num-worker: 1 tp: 4 @@ -5036,7 +5036,7 @@ qwen3.5-fp8-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/4p1d-dep4-dep16.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 16 @@ -5051,7 +5051,7 @@ qwen3.5-fp8-gb200-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/8p1d-dep4-dep16.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-8p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 16 @@ -5084,7 +5084,7 @@ dsv4-fp4-gb200-dynamo-sglang-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p1d-tp8-tp8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p1d-p-tp8-d-tp8-b8-c1-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -5100,7 +5100,7 @@ dsv4-fp4-gb200-dynamo-sglang-mtp: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p6d-dep8-tp8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p6d-p-tp8-ep8-dp8-d-tp8-b128-c32x64x128-mtp.yaml" decode: num-worker: 6 tp: 8 @@ -5115,7 +5115,7 @@ dsv4-fp4-gb200-dynamo-sglang-mtp: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-1p1d-dep8-dep16-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-1p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b1024-c1024-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -5130,7 +5130,7 @@ dsv4-fp4-gb200-dynamo-sglang-mtp: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-2p1d-dep8-dep16-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-2p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b2048-c2048-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -5145,7 +5145,7 @@ dsv4-fp4-gb200-dynamo-sglang-mtp: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-3p1d-dep8-dep16-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-3p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b4096-c3072-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -5160,7 +5160,7 @@ dsv4-fp4-gb200-dynamo-sglang-mtp: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-4p1d-dep8-dep16-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-4p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b6144-c6144-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -5175,7 +5175,7 @@ dsv4-fp4-gb200-dynamo-sglang-mtp: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-5p1d-dep8-dep16-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-5p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b16384-c8192-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -5190,7 +5190,7 @@ dsv4-fp4-gb200-dynamo-sglang-mtp: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-6p1d-dep8-dep16-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb200-fp4/8k1k/disagg-6p1d-p-tp8-ep8-dp8-d-tp16-ep16-dp16-b21504-c16384-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -5223,7 +5223,7 @@ dsv4-fp4-b300-dynamo-vllm: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b300-low-latency.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/b300-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp8-c1x32x64x128-stp.yaml" decode: num-worker: 1 tp: 8 @@ -5236,7 +5236,7 @@ dsv4-fp4-b300-dynamo-vllm: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b300-mid-curve-megamoe.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/b300-fp4/8k1k/disagg-1p1d-p-tp1-dp8-d-tp1-dp8-c256x1024-stp.yaml" decode: num-worker: 1 tp: 8 @@ -5249,7 +5249,7 @@ dsv4-fp4-b300-dynamo-vllm: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b300-high-tpt-megamoe.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/b300-fp4/8k1k/disagg-high-tpt-megamoe-stp.yaml" decode: num-worker: 1 tp: 8 @@ -5284,7 +5284,7 @@ dsv4-fp4-b300-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p1d-tp4-tp4-stp.yaml" decode: num-worker: 1 tp: 4 @@ -5298,7 +5298,7 @@ dsv4-fp4-b300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p1d-dep4-dep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -5312,7 +5312,7 @@ dsv4-fp4-b300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/8k1k/disagg-2p1d-dep4-dep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -5326,7 +5326,7 @@ dsv4-fp4-b300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/8k1k/disagg-4p1d-dep4-dep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -5340,7 +5340,7 @@ dsv4-fp4-b300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/8k1k/disagg-6p1d-dep4-dep8-stp.yaml" decode: num-worker: 1 tp: 8 @@ -5370,7 +5370,7 @@ dsv4-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p6d-dep4-tp4.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-1p6d-dep4-tp4-stp.yaml" decode: num-worker: 6 tp: 4 @@ -5383,7 +5383,7 @@ dsv4-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p9d-tep4-tp4.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-1p9d-tep4-tp4-stp.yaml" decode: num-worker: 9 tp: 4 @@ -5396,7 +5396,7 @@ dsv4-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-4p1d-dep4-dep8-24-c4096.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep8-24-c4096-stp.yaml" decode: num-worker: 1 tp: 8 @@ -5409,7 +5409,7 @@ dsv4-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-5p1d-dep4-dep8-28-c4096.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-5p1d-dep4-dep8-28-c4096-stp.yaml" decode: num-worker: 1 tp: 8 @@ -5422,7 +5422,7 @@ dsv4-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-6p1d-dep4-dep8-32-c4096.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-6p1d-dep4-dep8-32-c4096-stp.yaml" decode: num-worker: 1 tp: 8 @@ -5435,7 +5435,7 @@ dsv4-fp4-gb300-dynamo-vllm: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-7p2d-dep4-dep16.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/8k1k/disagg-7p2d-dep4-dep16-stp.yaml" decode: num-worker: 2 tp: 16 @@ -5466,7 +5466,7 @@ dsv4-fp4-b200-dynamo-sglang-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp8-d-tp8-b8-c1-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -5480,7 +5480,7 @@ dsv4-fp4-b200-dynamo-sglang-mtp: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p6d-p-tp8-ep8-dp8-d-tp8-b128-c32x64x128-mtp.yaml" decode: num-worker: 6 tp: 8 @@ -5494,7 +5494,7 @@ dsv4-fp4-b200-dynamo-sglang-mtp: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p1d-p-tp8-ep8-dp8-d-tp8-ep8-dp8-b1536-c256x1024-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -5508,7 +5508,7 @@ dsv4-fp4-b200-dynamo-sglang-mtp: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/8k1k/disagg-1p2d-p-tp8-ep8-dp8-d-tp8-ep8-dp8-b1024-c256-mtp.yaml" decode: num-worker: 2 tp: 8 @@ -5522,7 +5522,7 @@ dsv4-fp4-b200-dynamo-sglang-mtp: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b200-fp4/8k1k/disagg-5p3d-p-tp8-ep8-dp8-d-tp8-ep8-dp8-b2560-c6144-mtp.yaml" decode: num-worker: 3 tp: 8 @@ -5559,7 +5559,7 @@ dsv4-fp4-b200-dynamo-vllm-mtp: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-dep8-tp8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -5575,7 +5575,7 @@ dsv4-fp4-b200-dynamo-vllm-mtp: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p2d-dep8-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p2d-dep8-dep8-mtp.yaml" decode: num-worker: 2 tp: 1 @@ -5591,7 +5591,7 @@ dsv4-fp4-b200-dynamo-vllm-mtp: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/b200-fp4/8k1k/disagg-1p1d-dep8-dep8-mtp.yaml" decode: num-worker: 1 tp: 1 @@ -5607,7 +5607,7 @@ dsv4-fp4-b200-dynamo-vllm-mtp: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-2p1d-dep8-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/b200-fp4/8k1k/disagg-2p1d-dep8-dep8-mtp.yaml" decode: num-worker: 1 tp: 1 @@ -5623,7 +5623,7 @@ dsv4-fp4-b200-dynamo-vllm-mtp: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=1.79" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-3p1d-dep8-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/b200-fp4/8k1k/disagg-3p1d-dep8-dep8-mtp.yaml" decode: num-worker: 1 tp: 1 @@ -5654,7 +5654,7 @@ dsv4-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p4d-dep4-tep8-b1-eplb0-mtp0.yaml" decode: num-worker: 4 tp: 8 @@ -5668,7 +5668,7 @@ dsv4-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 4 @@ -5682,7 +5682,7 @@ dsv4-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b2-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 4 @@ -5696,7 +5696,7 @@ dsv4-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 4 @@ -5710,7 +5710,7 @@ dsv4-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b8-eplb0-mtp0.yaml" decode: num-worker: 5 tp: 4 @@ -5724,7 +5724,7 @@ dsv4-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen1dep32_batch4_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen1dep32_batch4_eplb384_mtp0.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p1d-dep4-dep32-b4-eplb384-mtp0.yaml" decode: num-worker: 1 tp: 32 @@ -5738,7 +5738,7 @@ dsv4-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb384_mtp0.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-2p1d-dep4-dep32-b8-eplb384-mtp0.yaml" decode: num-worker: 1 tp: 32 @@ -5752,7 +5752,7 @@ dsv4-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb384_mtp0.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep32-b16-eplb384-mtp0.yaml" decode: num-worker: 1 tp: 32 @@ -5766,7 +5766,7 @@ dsv4-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb384_mtp0.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-6p1d-dep4-dep32-b32-eplb384-mtp0.yaml" decode: num-worker: 1 tp: 32 @@ -5780,7 +5780,7 @@ dsv4-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb384_mtp0.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-5p1d-dep4-dep16-b64-eplb384-mtp0.yaml" decode: num-worker: 1 tp: 16 @@ -5794,7 +5794,7 @@ dsv4-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep8_batch256_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep8_batch256_eplb384_mtp0.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-6p1d-dep4-dep8-b256-eplb384-mtp0.yaml" decode: num-worker: 1 tp: 8 @@ -5808,7 +5808,7 @@ dsv4-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb384_mtp0.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep4-dep16-b128-eplb384-mtp0.yaml" decode: num-worker: 1 tp: 16 @@ -5822,7 +5822,7 @@ dsv4-fp4-gb300-dynamo-trt: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx10dep4_gen1dep8_batch512_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx10dep4_gen1dep8_batch512_eplb384_mtp0.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep4-dep8-b512-eplb384-mtp0.yaml" decode: num-worker: 1 tp: 8 @@ -5854,7 +5854,7 @@ dsv4-fp4-gb300-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p4d-dep4-tep8-b1-eplb0-mtp3.yaml" decode: num-worker: 4 tp: 8 @@ -5869,7 +5869,7 @@ dsv4-fp4-gb300-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b1-eplb0-mtp3.yaml" decode: num-worker: 5 tp: 4 @@ -5884,7 +5884,7 @@ dsv4-fp4-gb300-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b2-eplb0-mtp3.yaml" decode: num-worker: 5 tp: 4 @@ -5899,7 +5899,7 @@ dsv4-fp4-gb300-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-1p5d-dep4-tep4-b4-eplb0-mtp3.yaml" decode: num-worker: 5 tp: 4 @@ -5914,7 +5914,7 @@ dsv4-fp4-gb300-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch2_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch2_eplb384_mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-2p1d-dep4-dep32-b2-eplb384-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -5929,7 +5929,7 @@ dsv4-fp4-gb300-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx3dep4_gen1dep32_batch4_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx3dep4_gen1dep32_batch4_eplb384_mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-3p1d-dep4-dep32-b4-eplb384-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -5944,7 +5944,7 @@ dsv4-fp4-gb300-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb384_mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep32-b8-eplb384-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -5959,7 +5959,7 @@ dsv4-fp4-gb300-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep32_batch16_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep32_batch16_eplb384_mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-8p1d-dep4-dep32-b16-eplb384-mtp3.yaml" decode: num-worker: 1 tp: 32 @@ -5974,7 +5974,7 @@ dsv4-fp4-gb300-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb384_mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-6p1d-dep4-dep16-b32-eplb384-mtp3.yaml" decode: num-worker: 1 tp: 16 @@ -5989,7 +5989,7 @@ dsv4-fp4-gb300-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep8_batch128_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep8_batch128_eplb384_mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep4-dep8-b128-eplb384-mtp3.yaml" decode: num-worker: 1 tp: 8 @@ -6004,7 +6004,7 @@ dsv4-fp4-gb300-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx10dep4_gen1dep16_batch64_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx10dep4_gen1dep16_batch64_eplb384_mtp3.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep4-dep16-b64-eplb384-mtp3.yaml" decode: num-worker: 1 tp: 16 @@ -6019,7 +6019,7 @@ dsv4-fp4-gb300-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx9dep4_gen1dep8_batch256_eplb384_mtp1.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx9dep4_gen1dep8_batch256_eplb384_mtp1.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep4-dep8-b256-eplb384-mtp1.yaml" decode: num-worker: 1 tp: 8 @@ -6034,7 +6034,7 @@ dsv4-fp4-gb300-dynamo-trt-mtp: dp-attn: true additional-settings: # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep8_batch512_eplb384_mtp1.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep8_batch512_eplb384_mtp1.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/8k1k/disagg-12p1d-dep4-dep8-b512-eplb384-mtp1.yaml" decode: num-worker: 1 tp: 8 @@ -6065,7 +6065,7 @@ dsv4-fp4-gb300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-dep4-dep16-5-c1024-stp.yaml" decode: num-worker: 1 tp: 16 @@ -6079,7 +6079,7 @@ dsv4-fp4-gb300-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-tp4-tp4-2-c1-stp.yaml" decode: num-worker: 1 tp: 4 @@ -6094,7 +6094,7 @@ dsv4-fp4-gb300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-15p1d-dep4-dep12-18-c12000-stp.yaml" decode: num-worker: 1 tp: 12 @@ -6108,7 +6108,7 @@ dsv4-fp4-gb300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-14p1d-dep4-dep16-18-c8192-stp.yaml" decode: num-worker: 1 tp: 16 @@ -6122,7 +6122,7 @@ dsv4-fp4-gb300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-12p1d-dep4-dep24-18-c3000-stp.yaml" decode: num-worker: 1 tp: 24 @@ -6136,7 +6136,7 @@ dsv4-fp4-gb300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-10p1d-dep4-dep32-18-c2500-stp.yaml" decode: num-worker: 1 tp: 32 @@ -6150,7 +6150,7 @@ dsv4-fp4-gb300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-8p1d-dep4-dep40-18-c2048-stp.yaml" decode: num-worker: 1 tp: 40 @@ -6183,7 +6183,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-d-tp4-b8-c1-mtp.yaml" decode: num-worker: 1 tp: 4 @@ -6199,7 +6199,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p6d-p-tp4-ep4-dp4-d-tp4-b128-c8x32x64-mtp.yaml" decode: num-worker: 6 tp: 4 @@ -6214,7 +6214,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b3072-c256-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -6229,7 +6229,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b3072-c256-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -6244,7 +6244,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-2p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b3072-c512-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -6259,7 +6259,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp8-ep8-dp8-b3072-c1024-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -6274,7 +6274,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -6289,7 +6289,7 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -6327,7 +6327,7 @@ dsv4-fp4-b300-dynamo-sglang-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p1d-tp4-tp4-mtp.yaml" decode: num-worker: 1 tp: 4 @@ -6342,7 +6342,7 @@ dsv4-fp4-b300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p6d-dep4-tp4-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p6d-dep4-tp4-mtp.yaml" decode: num-worker: 6 tp: 4 @@ -6357,7 +6357,7 @@ dsv4-fp4-b300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p1d-dep4-dep8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -6372,7 +6372,7 @@ dsv4-fp4-b300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p2d-dep4-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/8k1k/disagg-1p2d-dep4-dep8-mtp.yaml" decode: num-worker: 2 tp: 8 @@ -6387,7 +6387,7 @@ dsv4-fp4-b300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/8k1k/disagg-2p1d-dep4-dep8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -6402,7 +6402,7 @@ dsv4-fp4-b300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/8k1k/disagg-4p1d-dep4-dep8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -6417,7 +6417,7 @@ dsv4-fp4-b300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/8k1k/disagg-6p1d-dep4-dep8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -6432,7 +6432,7 @@ dsv4-fp4-b300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b300-8p1d-dep4-dep8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/8k1k/disagg-8p1d-dep4-dep8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -6500,7 +6500,7 @@ qwen3.5-fp4-gb300-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x4x8x16x32x64x256-stp.yaml" decode: num-worker: 1 tp: 4 @@ -6517,7 +6517,7 @@ qwen3.5-fp4-gb300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-5p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b4096-c2048-stp.yaml" decode: num-worker: 1 tp: 16 @@ -6534,7 +6534,7 @@ qwen3.5-fp4-gb300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml" decode: num-worker: 1 tp: 16 @@ -6551,7 +6551,7 @@ qwen3.5-fp4-gb300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b5120-c5120-stp.yaml" decode: num-worker: 1 tp: 16 @@ -6581,7 +6581,7 @@ qwen3.5-fp4-gb300-dynamo-trt: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen2_tep8_batch16_eplb0_mtp0_42.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p2d-tep8-b16-eplb0-mtp0-c42.yaml" decode: num-worker: 2 tp: 8 @@ -6595,7 +6595,7 @@ qwen3.5-fp4-gb300-dynamo-trt: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_8.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0-c8.yaml" decode: num-worker: 4 tp: 8 @@ -6609,7 +6609,7 @@ qwen3.5-fp4-gb300-dynamo-trt: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml" decode: num-worker: 4 tp: 8 @@ -6623,7 +6623,7 @@ qwen3.5-fp4-gb300-dynamo-trt: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx4_gen1_dep16_batch16_eplb0_mtp0_282.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep16-b16-eplb0-mtp0-c282.yaml" decode: num-worker: 1 tp: 16 @@ -6637,7 +6637,7 @@ qwen3.5-fp4-gb300-dynamo-trt: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx4_gen3_tep8_batch64_eplb0_mtp0_210.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-4p3d-tep8-b64-eplb0-mtp0-c210.yaml" decode: num-worker: 3 tp: 8 @@ -6651,7 +6651,7 @@ qwen3.5-fp4-gb300-dynamo-trt: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx7_gen1_dep16_batch32_eplb0_mtp0_563.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep16-b32-eplb0-mtp0-c563.yaml" decode: num-worker: 1 tp: 16 @@ -6665,7 +6665,7 @@ qwen3.5-fp4-gb300-dynamo-trt: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx10_gen1_dep8_batch256_eplb0_mtp0_2150.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp0-c2150.yaml" decode: num-worker: 1 tp: 8 @@ -6679,7 +6679,7 @@ qwen3.5-fp4-gb300-dynamo-trt: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx11_gen1_dep16_batch64_eplb0_mtp0_1076.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-11p1d-dep16-b64-eplb0-mtp0-c1076.yaml" decode: num-worker: 1 tp: 16 @@ -6693,7 +6693,7 @@ qwen3.5-fp4-gb300-dynamo-trt: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx16_gen1_dep16_batch128_eplb0_mtp0_2253.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-16p1d-dep16-b128-eplb0-mtp0-c2253.yaml" decode: num-worker: 1 tp: 16 @@ -6707,7 +6707,7 @@ qwen3.5-fp4-gb300-dynamo-trt: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/stp/ctx24_gen1_dep16_batch512_eplb0_mtp0_8192.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-24p1d-dep16-b512-eplb0-mtp0-c8192-stp-sweep.yaml" decode: num-worker: 1 tp: 16 @@ -6742,7 +6742,7 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen2_tep8_batch8_eplb0_mtp3_20.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p2d-tep8-b8-eplb0-mtp3-c20.yaml" decode: num-worker: 2 tp: 8 @@ -6757,7 +6757,7 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_12.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3-c12.yaml" decode: num-worker: 4 tp: 8 @@ -6772,7 +6772,7 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx1_gen4_tep8_batch2_eplb0_mtp3_8.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b2-eplb0-mtp3-c8.yaml" decode: num-worker: 4 tp: 8 @@ -6787,7 +6787,7 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx4_gen3_tep8_batch32_eplb0_mtp3_126.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-4p3d-tep8-b32-eplb0-mtp3-c126.yaml" decode: num-worker: 3 tp: 8 @@ -6802,7 +6802,7 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx5_gen1_dep16_batch8_eplb0_mtp3_154.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-5p1d-dep16-b8-eplb0-mtp3-c154.yaml" decode: num-worker: 1 tp: 16 @@ -6817,7 +6817,7 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx7_gen1_dep16_batch32_eplb0_mtp3_666.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep16-b32-eplb0-mtp3-c666.yaml" decode: num-worker: 1 tp: 16 @@ -6832,7 +6832,7 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx11_gen1_dep8_batch128_eplb0_mtp3_1229.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-11p1d-dep8-b128-eplb0-mtp3-c1229.yaml" decode: num-worker: 1 tp: 8 @@ -6847,7 +6847,7 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx17_gen2_dep8_batch64_eplb0_mtp3_1126.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-17p2d-dep8-b64-eplb0-mtp3-c1126.yaml" decode: num-worker: 2 tp: 8 @@ -6862,7 +6862,7 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx26_gen1_dep16_batch256_eplb0_mtp2_4301.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-26p1d-dep16-b256-eplb0-mtp2-c4301.yaml" decode: num-worker: 1 tp: 16 @@ -6877,7 +6877,7 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/8k1k/mtp/ctx24_gen1_dep16_batch512_eplb0_mtp0_8192.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/8k1k/disagg-24p1d-dep16-b512-eplb0-mtp0-c8192-mtp-sweep.yaml" decode: num-worker: 1 tp: 16 @@ -6911,7 +6911,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p1d-dep1-tep2-c44-b8-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 2 @@ -6928,7 +6928,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml" decode: num-worker: 7 tp: 8 @@ -6945,7 +6945,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 1 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p2d-dep1-tep2-c52-b4-mtp-kvoffload.yaml" decode: num-worker: 2 tp: 2 @@ -6962,7 +6962,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 2 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-2p3d-tep2-tep8-c96-b128-mtp-kvoffload.yaml" decode: num-worker: 3 tp: 8 @@ -6979,7 +6979,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p1d-dep4-dep16-c565-b8-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 @@ -6996,7 +6996,7 @@ qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/qwen3.5/trtllm/gb300-fp4/agentx/disagg-3p2d-dep4-dep4-c704-b32-mtp-kvoffload.yaml" decode: num-worker: 2 tp: 4 @@ -7027,7 +7027,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c1-mtp-hicache-jid2530006.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -7039,7 +7039,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c24-mtp-hicache-jid2530012.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -7051,7 +7051,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c32-mtp-hicache-jid2530013.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -7063,7 +7063,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c40-mtp-hicache-jid2530015.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -7075,7 +7075,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c48-mtp-hicache-jid2530017.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -7087,7 +7087,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c52-mtp-hicache-jid2527406.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -7099,7 +7099,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c64-mtp-hicache-jid2527410.yaml" # The seven disaggregated frontier points use six TP4 shapes plus one TP2 # shape. Stable X-Dynamo-Session-ID affinity replaces the removed conv-aware # routing message path. @@ -7128,7 +7128,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml" decode: num-worker: 1 tp: 4 @@ -7144,7 +7144,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml" decode: num-worker: 1 tp: 4 @@ -7160,7 +7160,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml" decode: num-worker: 1 tp: 4 @@ -7176,7 +7176,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml" decode: num-worker: 1 tp: 4 @@ -7192,7 +7192,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml" decode: num-worker: 1 tp: 4 @@ -7208,7 +7208,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml" decode: num-worker: 1 tp: 4 @@ -7224,7 +7224,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml" decode: num-worker: 1 tp: 2 @@ -7263,7 +7263,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=4.41" - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p2d-pp4-dep4-c704-mtp-hicache-nightly-20260831.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p2d-pp4-dep4-c704-mtp-hicache-nightly-c20260831.yaml" decode: num-worker: 2 tp: 4 @@ -7283,7 +7283,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.97" - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p4d-pp4-dep4-c565-mtp-hicache-nightly-20260831.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-3p4d-pp4-dep4-c565-mtp-hicache-nightly-c20260831.yaml" decode: num-worker: 4 tp: 4 @@ -7318,7 +7318,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=4.8" - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b2-mtp-hicache-nightly-20260831.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b2-mtp-hicache-nightly-c20260831.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -7333,7 +7333,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=4.8" - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c44-b1-mtp-hicache-nightly-20260831.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp2-c44-b1-mtp-hicache-nightly-c20260831.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -7348,7 +7348,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=4.8" - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp8-c7-b1-mtp-hicache-nightly-20260831.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-tp8-c7-b1-mtp-hicache-nightly-c20260831.yaml" # ---------- 1k1k high-throughput (wide-EP decode, EAGLE MTP) ---------- qwen3.5-fp8-b300-sglang-agentic-mtp: @@ -7463,7 +7463,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml" decode: num-worker: 6 tp: 4 @@ -7479,7 +7479,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p6d-dep4-tp4.yaml" decode: num-worker: 6 tp: 4 @@ -7496,7 +7496,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-4p1d-dep4-dep8-24-c4096.yaml" decode: num-worker: 1 tp: 8 @@ -7758,7 +7758,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=4.41" - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp.yaml" - spec-decoding: mtp kv-offloading: none conc-list: [2] @@ -7771,7 +7771,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=4.41" - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-no-symm.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-no-symm.yaml" - spec-decoding: mtp kv-offloading: none conc-list: [24] @@ -7784,7 +7784,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=4.41" - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-parity.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp4-mtp-parity.yaml" # Keep only the reproducible TP2/EP2 HiCache K5 transition point; C28 # was dominated by the published C28 result in the official PR sweep. - spec-decoding: mtp @@ -7800,7 +7800,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=4.41" - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache.yaml" # Keep the published C32 interactivity anchor. The tighter capacity # layout becomes Pareto-relevant at C40 and remains deployable at C48. - spec-decoding: mtp @@ -7816,7 +7816,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.39" - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache-k3-baseline.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-k3-baseline.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } @@ -7830,7 +7830,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.39" - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache-cap48.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-tp2ep2-mtp-hicache-cap48.yaml" minimaxm3-fp4-b300-vllm-agentic-mtp: image: vllm/vllm-openai:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45 model: nvidia/MiniMax-M3-NVFP4 @@ -7875,8 +7875,8 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 4 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tep4-tp4-c1-eval-agentic.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1.yaml" + - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tep4-tp4-c1-eval.yaml" decode: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -7888,8 +7888,8 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-dep4-tp4-c24-eval-agentic.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24.yaml" + - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-dep4-tp4-c24-eval.yaml" decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -7901,8 +7901,8 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p1d-tp2-tp4-c20-c24-eval-agentic.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24.yaml" + - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p1d-tp2-tp4-c20-c24-eval.yaml" decode: { num-worker: 1, tp: 4, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -7914,8 +7914,8 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/1p3d-tp2-tp2-c48-eval-agentic.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48.yaml" + - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-1p3d-tp2-tp2-c48-eval.yaml" decode: { num-worker: 3, tp: 2, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -7927,8 +7927,8 @@ minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/minimax-m3/agentic/gb300-fp4/2p5d-tp2-tp2-c120-eval-agentic.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120.yaml" + - "EVAL_CONFIG_FILE=recipes/minimaxm3/vllm/gb300-fp4/agentx/disagg-2p5d-tp2-tp2-c120-eval.yaml" decode: { num-worker: 5, tp: 2, ep: 1, dp-attn: false } minimaxm3-fp4-b300-trtllm-agentic-mtp: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1 @@ -8047,7 +8047,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic-nightly-native.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-nightly-native.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" - spec-decoding: mtp @@ -8061,7 +8061,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic-nightly-native.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-nightly-native.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" - spec-decoding: mtp @@ -8074,7 +8074,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp8-agentic-nightly-native.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp8-nightly-native.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" @@ -8104,7 +8104,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: ep: 4 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-tp4-tp8-c1-agentic.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp8-c1.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: false } @@ -8118,7 +8118,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: ep: 4 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p2d-tp4-tp4-c8-c16-agentic.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p2d-tp4-tp4-c8-c16.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" decode: { num-worker: 2, tp: 4, ep: 4, dp-attn: false } @@ -8132,7 +8132,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: ep: 4 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-tp4-tp4-c24-agentic.yaml" + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p1d-tp4-tp4-c24.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" decode: { num-worker: 1, tp: 4, ep: 4, dp-attn: false } @@ -8162,7 +8162,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c5-b1-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c5-b5-eagle3.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" - spec-decoding: mtp @@ -8176,7 +8176,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c10-b1-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c10-b10-eagle3.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" - spec-decoding: mtp @@ -8190,7 +8190,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c15-b1-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c15-b15-eagle3.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" - spec-decoding: mtp @@ -8204,7 +8204,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c20-b1-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c20-b20-eagle3.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" - spec-decoding: mtp @@ -8218,7 +8218,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c25-b1-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c25-b25-eagle3.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" - spec-decoding: mtp @@ -8232,7 +8232,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c30-b1-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c30-b30-eagle3.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" - spec-decoding: mtp @@ -8246,7 +8246,7 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic/dynamo-agg-gb200-tp4-c40-b1-eagle3.yaml" + - "CONFIG_FILE=recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-tp4-c40-b40-eagle3.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" @@ -8276,7 +8276,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-mtp.yaml" - spec-decoding: mtp conc-list: [8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80] num-nodes: 2 @@ -8288,7 +8288,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-dep8-mtp.yaml" dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 model: deepseek-ai/DeepSeek-V4-Pro @@ -8314,7 +8314,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -8331,7 +8331,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep8-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -8364,7 +8364,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml" - spec-decoding: mtp conc-list: [4] num-nodes: 2 @@ -8376,7 +8376,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp8-mtp.yaml" - spec-decoding: mtp conc-list: [1, 2, 4, 6, 8] num-nodes: 1 @@ -8388,7 +8388,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/agg-tp4-mtp.yaml" dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f model: deepseek-ai/DeepSeek-V4-Pro @@ -8419,7 +8419,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep12-c1152-mtp.yaml" decode: num-worker: 1 tp: 12 @@ -8438,7 +8438,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c1024-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -8457,7 +8457,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep8-c256-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -8476,7 +8476,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c512-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -8493,7 +8493,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tp8-c4-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml" decode: num-worker: 4 tp: 8 @@ -8512,7 +8512,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c128-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c128-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -8531,7 +8531,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep16-c256-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p1d-dep4-dep16-c256-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -8569,7 +8569,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-c4-mtp2-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml" - spec-decoding: mtp conc-list: [4] num-nodes: 2 @@ -8581,7 +8581,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-c4-mtp2-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c4-mtp3.yaml" - spec-decoding: mtp conc-list: [8] num-nodes: 2 @@ -8593,7 +8593,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-c8-mtp2-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-tp8-c8-mtp3.yaml" dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f model: deepseek-ai/DeepSeek-V4-Pro @@ -8624,7 +8624,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-c576-mtp2-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep12-c576-mtp3.yaml" decode: num-worker: 1 tp: 12 @@ -8643,7 +8643,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep16-c512-mtp2-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c512-mtp3.yaml" decode: num-worker: 1 tp: 16 @@ -8662,7 +8662,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-c256-mtp2-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c256-mtp3.yaml" decode: num-worker: 1 tp: 8 @@ -8681,7 +8681,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-c128-mtp2-agentic.yaml" + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-1p1d-dep8-dep8-c128-mtp3.yaml" decode: num-worker: 1 tp: 8 @@ -8723,7 +8723,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp16-latency.yaml" # Balanced: multi_node_tep strategy, TEP16 across four GB200 nodes. # https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200&nodes=4&strategy=multi_node_tep - spec-decoding: mtp @@ -8735,7 +8735,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: ep: 16 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-tep16-balanced.yaml" # Throughput oriented: official multi_node_dep strategy, DEP16 across # four GB200 nodes (TP4 x DP4 = EP16, one local DP rank per node). # The recipe allows up to 3600s for full-context saturation warmup to drain. @@ -8749,7 +8749,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: ep: 16 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16.yaml" # High-concurrency DEP16 with vLLM Simple CPU KV offloading. c384 # exercises 384 of the 393 AgentX trajectories and remains below the # aggregate max-num-seqs capacity of 512 (128 per DP rank). @@ -8764,7 +8764,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic: ep: 16 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-dep16-vllm-simple-offload.yaml" # Kimi-K3 GB200 TP16/DCP16 profiles using Mooncake DRAM offload. kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-agg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef @@ -8793,7 +8793,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-dspark4-maxseq2-mooncake-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml" kimik3-fp4-gb200-dynamo-vllm-agentic-mooncake-dcp16-agg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef @@ -8819,7 +8819,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-mooncake-dcp16-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-nospec-mooncake-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-nospec-mooncake.yaml" dsv4-fp4-gb300-dynamo-trt-agentx: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc24 @@ -8846,7 +8846,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p4d-dep4-tep8-c4-b1-mtp.yaml" - "SLURM_PARTITION=batch_1" decode: num-worker: 4 @@ -8863,7 +8863,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p6d-dep4-tep4-c24-b4-mtp.yaml" - "SLURM_PARTITION=batch_1" decode: num-worker: 6 @@ -8880,7 +8880,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-1p1d-dep8-dep32-c388-b4-mtp.yaml" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -8897,7 +8897,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-2p1d-dep8-dep32-c736-b8-mtp.yaml" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -8914,7 +8914,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1152-b32-mtp.yaml" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -8931,7 +8931,7 @@ dsv4-fp4-gb300-dynamo-trt-agentx: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp.yaml" + - "CONFIG_FILE=recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-5p1d-dep8-dep16-c2626-b96-mtp.yaml" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -8966,7 +8966,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c16-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c16.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - spec-decoding: mtp @@ -8982,7 +8982,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c32-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c32.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - kv-offloading: dram @@ -8997,7 +8997,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c48-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml" - kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } conc-list: [72] @@ -9010,7 +9010,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c72-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c72.yaml" - kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } conc-list: [96] @@ -9023,7 +9023,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c96-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml" dsv4-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:v0.5.19-cu130@sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9 model: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -9048,7 +9048,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp8-mtp-lowlatency.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp8-mtp.yaml" - search-space: - spec-decoding: draft_model conc-list: [8] @@ -9061,7 +9061,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-lowlatency.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/agg-tp4-mtp.yaml" dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-cu13-20260829-89816a21 model: deepseek-ai/DeepSeek-V4-Pro-0813 @@ -9089,7 +9089,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c480-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-1p1d-dep8-dep16-c480-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 @@ -9107,7 +9107,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c960-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-2p1d-dep8-dep16-c960-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 @@ -9125,7 +9125,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c1440-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-3p1d-dep8-dep16-c1440-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 @@ -9143,7 +9143,7 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.77" - - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/disagg-gb300-8p4d-dep8-dep16-c1920-mtp-kvoffload.yaml" + - "CONFIG_FILE=recipes/dsv4/sglang/gb300-fp4/agentx/disagg-4p1d-dep8-dep16-c1920-mtp-kvoffload.yaml" decode: num-worker: 1 tp: 16 @@ -9174,7 +9174,7 @@ qwen3.5-fp8-gb300-dynamo-sglang: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-tp4-tp4-stp.yaml" decode: num-worker: 1 tp: 4 @@ -9189,7 +9189,7 @@ qwen3.5-fp8-gb300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/8k1k/4p1d-dep4-dep16.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 16 @@ -9204,7 +9204,7 @@ qwen3.5-fp8-gb300-dynamo-sglang: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/8k1k/8p1d-dep4-dep16.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-8p1d-dep4-dep16-stp.yaml" decode: num-worker: 1 tp: 16 @@ -9239,7 +9239,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml" decode: num-worker: 2 tp: 8 @@ -9255,7 +9255,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml" decode: num-worker: 2 tp: 8 @@ -9271,7 +9271,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/disagg-2p2d-pcp8-tp8-dp8-mtp.yaml" decode: num-worker: 2 tp: 8 @@ -9307,7 +9307,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-1p1d-hicache: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml" decode: num-worker: 1 tp: 8 @@ -9324,7 +9324,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-1p1d-hicache: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml" decode: num-worker: 1 tp: 8 @@ -9341,7 +9341,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-1p1d-hicache: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml" decode: num-worker: 1 tp: 8 @@ -9433,7 +9433,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c1-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c1-mtp.yaml" - spec-decoding: mtp conc-list: [4] kv-offloading: dram @@ -9445,7 +9445,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c4-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c4-mtp.yaml" - spec-decoding: mtp conc-list: [8] kv-offloading: dram @@ -9457,7 +9457,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/b200-fp4/agentic/agg-b200-tp8-c8-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/agg-tp8-c8-mtp.yaml" glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-20260910-00840301 @@ -9485,7 +9485,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p4d-dep8-tp4-c48-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml" decode: num-worker: 4 tp: 4 @@ -9502,7 +9502,7 @@ glm5.2-fp4-b200-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/b200-fp4/agentic/disagg-b200-1p1d-dep8-dep8-c64-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/b200-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -9535,7 +9535,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-agg: dp-attn: false additional-settings: # Shared deployment GPU power telemetry. - - "CONFIG_FILE=recipes/sglang/glm5.2/gb200-fp4/agentic/glm5.2-agentx-agg.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml" glm5.2-fp4-gb200-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:v0.5.17-cu130 model: nvidia/GLM-5.2-NVFP4 @@ -9561,7 +9561,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/gb200-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_frontier[0]" + - "CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml:zip_override_mtp_agentx_frontier[0]" decode: num-worker: 1 tp: 4 @@ -9599,7 +9599,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 8 dp-attn: true additional-settings: - - CONFIG_FILE=recipes/sglang/glm5.2/gb200-fp4/agentic/disagg-gb200-1p6d-dep8-tp4-c45-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p6d-dep8-tp4-c45-mtp.yaml decode: num-worker: 6 tp: 4 @@ -9617,7 +9617,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 8 dp-attn: true additional-settings: - - CONFIG_FILE=recipes/sglang/glm5.2/gb200-fp4/agentic/disagg-gb200-1p4d-dep8-tp4-c48-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-1p4d-dep8-tp4-c48-mtp.yaml decode: num-worker: 4 tp: 4 @@ -9635,7 +9635,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp: ep: 8 dp-attn: true additional-settings: - - CONFIG_FILE=recipes/sglang/glm5.2/gb200-fp4/agentic/disagg-gb200-2p1d-dep8-dep16-c128-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-2p1d-dep8-dep16-c128-mtp.yaml decode: num-worker: 1 tp: 16 @@ -9673,7 +9673,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/sglang/glm5.2/gb200-fp4/agentic/agg-gb200-tp8-c2-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c2-mtp.yaml - spec-decoding: mtp kv-offloading: dram kv-offload-backend: @@ -9687,7 +9687,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/sglang/glm5.2/gb200-fp4/agentic/agg-gb200-tp8-c4-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c4-mtp.yaml - spec-decoding: mtp kv-offloading: dram kv-offload-backend: @@ -9701,7 +9701,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: - - CONFIG_FILE=recipes/sglang/glm5.2/gb200-fp4/agentic/agg-gb200-tp8-c8-mtp.yaml + - CONFIG_FILE=recipes/glm5.2/sglang/gb200-fp4/agentx/agg-tp8-c8-mtp.yaml glm5.2-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 model: nvidia/GLM-5.2-NVFP4 @@ -9728,7 +9728,7 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx-agg.yaml" + - "CONFIG_FILE=recipes/glm5.2/sglang/gb300-fp4/agentx/agg.yaml" glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 model: nvidia/GLM-5.2-NVFP4 @@ -9755,7 +9755,7 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_lowlat[0]" + - "CONFIG_FILE=recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-mtp-variants.yaml:zip_override_mtp_agentx_lowlat[0]" decode: num-worker: 2 tp: 4 @@ -9772,7 +9772,7 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_lowlat[1]" + - "CONFIG_FILE=recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-mtp-variants.yaml:zip_override_mtp_agentx_lowlat[1]" decode: num-worker: 4 tp: 4 @@ -9789,7 +9789,7 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_lowlat[2]" + - "CONFIG_FILE=recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-mtp-variants.yaml:zip_override_mtp_agentx_lowlat[2]" decode: num-worker: 6 tp: 4 @@ -9806,7 +9806,7 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_hightpt[0]" + - "CONFIG_FILE=recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-mtp-variants.yaml:zip_override_mtp_agentx_hightpt[0]" decode: num-worker: 1 tp: 16 @@ -9822,7 +9822,7 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_hightpt[1]" + - "CONFIG_FILE=recipes/glm5.2/sglang/gb300-fp4/agentx/disagg-mtp-variants.yaml:zip_override_mtp_agentx_hightpt[1]" decode: num-worker: 1 tp: 16 @@ -9856,7 +9856,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tp8-c1-b1-mtp5.yaml" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -9873,7 +9873,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p4d-tep4-c30-b2-mtp5.yaml" - "SLURM_PARTITION=batch_1" decode: num-worker: 4 @@ -9890,7 +9890,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-1p1d-tep8-c20-b5-mtp5.yaml" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -9907,7 +9907,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-3p4d-tep4-c60-b5-mtp5.yaml" - "SLURM_PARTITION=batch_1" decode: num-worker: 4 @@ -9924,7 +9924,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c260-b16-mtp3.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-5p1d-dep16-c260-b16-mtp3.yaml" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -9941,7 +9941,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-4p1d-dep8-c227-b16-mtp3.yaml" + - "CONFIG_FILE=recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-4p1d-dep8-c227-b16-mtp3.yaml" - "SLURM_PARTITION=batch_1" decode: num-worker: 1 @@ -9974,7 +9974,7 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b128-c1x2x8-mtp.yaml" decode: num-worker: 1 tp: 4 @@ -9989,7 +9989,7 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: ep: 8 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -10004,7 +10004,7 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_0.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -10019,7 +10019,7 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_1.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -10034,7 +10034,7 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_2.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -10049,7 +10049,7 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_3.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -10064,7 +10064,7 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_4.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -10103,7 +10103,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake.yaml" decode: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -10118,7 +10118,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p2d-dcp8-dcp8-dspark4-mooncake.yaml" decode: { num-worker: 2, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -10133,7 +10133,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark7-mooncake.yaml" decode: { num-worker: 3, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram @@ -10148,7 +10148,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb300-fp4/agentx/disagg-1p3d-dcp8-dcp8-dspark4-mooncake.yaml" decode: { num-worker: 3, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: @@ -10178,7 +10178,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark7-maxseq2-mooncake.yaml" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } @@ -10193,7 +10193,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark4-mooncake-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/gb300-fp4/agentx/agg-dcp8-dspark4-mooncake.yaml" # Kimi-K3 MXFP4 B200 aggregated vLLM (TP8 x PP2, 2 nodes / 16 GPUs), agentic @@ -10235,7 +10235,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c1-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c1.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - spec-decoding: mtp @@ -10250,7 +10250,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c4-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c4.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - spec-decoding: mtp @@ -10265,7 +10265,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c8-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c8.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - spec-decoding: mtp @@ -10280,7 +10280,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c14-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c14.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - spec-decoding: mtp @@ -10295,7 +10295,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c24-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c24.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" - spec-decoding: mtp @@ -10310,7 +10310,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c48-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c48.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" - kv-offloading: none @@ -10324,7 +10324,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c96-agentic.yaml" + - "CONFIG_FILE=recipes/kimik3/vllm/b200-fp4/agentx/agg-tp8pp2-mooncake-c96.yaml" qwen3.5-fp8-gb300-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -10350,7 +10350,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp4-d-tp4-b1024-c1x2x8-mtp.yaml" decode: num-worker: 1 tp: 4 @@ -10365,7 +10365,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: ep: 8 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-1p1d-p-tp8-ep8-d-tp8-ep8-b1024-c32x48x80-mtp.yaml" decode: num-worker: 1 tp: 8 @@ -10380,7 +10380,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_0.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-3p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c480-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -10395,7 +10395,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_1.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-4p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c768-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -10410,7 +10410,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_2.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-6p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b1024-c1280-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -10425,7 +10425,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_3.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-7p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1344-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -10440,7 +10440,7 @@ qwen3.5-fp8-gb300-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_4.yaml" + - "CONFIG_FILE=recipes/qwen3.5/sglang/gb300-fp8/8k1k/disagg-8p1d-p-tp4-ep4-dp4-d-tp16-ep16-dp16-b2048-c1920x2304-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -10537,7 +10537,7 @@ glm5.1-fp8-b200-tilert-agentic: additional-settings: - "PREFILL_IMAGE=vllm/vllm-openai:v0.26.0" - "PREFILL_NODES=1" - - "CONFIG_FILE=recipes/tilert/glm5.1/b200-fp8/agentic/disagg-1p1d-tp8-mtp.yaml" + - "CONFIG_FILE=recipes/glm5.1/tilert/b200-fp8/agentx/disagg-1p1d-tp8-mtp.yaml" decode: num-worker: 1 tp: 8 diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 659210488f..9f688bcb1d 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -21,6 +21,18 @@ Use this page for benchmark configuration, recipe, image, and runner changes. It | [`perf-changelog.yaml`](../perf-changelog.yaml) | Append-only benchmark trigger log | | [`AGENTS.md`](../AGENTS.md) | Repository-wide config, MTP, changelog, and sweep rules | +## Dependency submodules + +Git records the exact dependency commits. [`.gitmodules`](../.gitmodules) defines the repositories: AIPerf at `utils/aiperf`, NVIDIA srt-slurm at `utils/srt-slurm`. TileRT is a documented manual fork checkout in `setup_srt_slurm()`, not a separate submodule. + +Initialize them before running benchmarks locally: + +```bash +git submodule update --init +``` + +To upgrade, fetch and check out the desired commit inside the relevant submodule, then commit the updated submodule pointer in InferenceX. Benchmark workflows already initialize submodules. Slurm launchers make a local Git clone for each job so recipe staging and runtime writes do not modify the submodule, and record the actual commit for result provenance. NVIDIA setup clones locally; TileRT setup fetches its pinned fork commit over the network. + ## Procedure index 1. [Prepare a worktree](#prepare-a-worktree) @@ -139,7 +151,7 @@ Only fixed 8192/1024 `glm5.1-fp8-b200-tilert` requires native power. TileRT runs Mapping source: [`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md). Checked-in recipes: [`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/). 1. Locate the exact upstream [NVIDIA/srt-slurm](https://github.com/NVIDIA/srt-slurm) recipe and record its commit-pinned source path. -2. Stage the YAML under the matching checked-in recipe tree. Read the closest sibling and selected cluster launcher. +2. Stage the YAML under `benchmarks/multi_node/srt-slurm-recipes///-//`, following the naming rules in `RECIPES.md`. Read the closest sibling and selected cluster launcher. 3. Map source fields to the master search-space entry: resource worker counts → `num-worker`, TP/EP/DP-attention → worker topology, benchmark concurrencies → `conc-list`, and recipe path → `additional-settings: ["CONFIG_FILE=..."]`. 4. Add/update the matching [`nvidia-master.yaml`](../configs/nvidia-master.yaml) entry in the same change. Keep worker counts, TP/PP/EP/DCP/PCP, hardware, router, transfer engine, and concurrency labels synchronized. 5. For an image bump, make recipe `model.container` exactly equal master `image`. The launcher uses the master image as the container-alias key. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index d554b721c2..cc38a73534 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -21,6 +21,18 @@ | [`perf-changelog.yaml`](../perf-changelog.yaml) | 只允许追加的基准触发日志 | | [`AGENTS.md`](../AGENTS.md) | 仓库级配置、MTP、changelog 和 sweep 规则 | +## 依赖子模块 + +Git 记录依赖的精确提交版本。[`.gitmodules`](../.gitmodules) 定义各仓库：AIPerf 位于 `utils/aiperf`，NVIDIA srt-slurm 位于 `utils/srt-slurm`。TileRT 由 `setup_srt_slurm()` 手动检出已记录的分支仓库，不是独立子模块。 + +本地运行基准测试前，先初始化子模块： + +```bash +git submodule update --init +``` + +升级时，在对应子模块中获取并检出目标提交，再将更新后的子模块指针提交到 InferenceX。基准测试工作流已配置为自动初始化子模块。Slurm 启动器为每个作业创建本地 Git 克隆，避免配方准备和运行时写入修改子模块，并记录实际提交以供结果溯源。NVIDIA 启动器使用本地克隆；TileRT 启动器通过网络获取固定的分支提交。 + ## 规程索引 1. [准备 worktree](#准备-worktree) @@ -137,7 +149,7 @@ B200 Nscale 的 GLM-5.1 可用 `MODEL_PATH` 指定已有共享权重，覆盖默 映射来源：[`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md)。检入的配方：[`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/)。 1. 定位精确的上游 [NVIDIA/srt-slurm](https://github.com/NVIDIA/srt-slurm) 配方，并记录固定到 commit 的来源路径。 -2. 将 YAML 暂存到匹配的检入配方目录。阅读最接近的同类项和所选集群 launcher。 +2. 将 YAML 放在 `benchmarks/multi_node/srt-slurm-recipes///-//` 下，遵循 `RECIPES_zh.md` 中的命名规范。阅读最接近的同类项和所选集群 launcher。 3. 将来源字段映射到主配置搜索空间条目：资源 worker 数 → `num-worker`；TP/EP/DP-attention → worker 拓扑；基准并发 → `conc-list`；配方路径 → `additional-settings: ["CONFIG_FILE=..."]`。 4. 在同一变更中添加/更新匹配的 [`nvidia-master.yaml`](../configs/nvidia-master.yaml) 条目。同步 worker 数、TP/PP/EP/DCP/PCP、hardware、router、传输引擎和并发标签。 5. 更新镜像时，使配方 `model.container` 与主配置 `image` 完全相同；launcher 使用主配置镜像作为 container alias key。 diff --git a/infx/matrix/generate.py b/infx/matrix/generate.py index abadad2cec..83108a2231 100644 --- a/infx/matrix/generate.py +++ b/infx/matrix/generate.py @@ -257,12 +257,27 @@ def recipe_node_count(prefill: dict, decode: dict) -> int | None: # config topology remains the best available scheduling estimate. return None - resources = yaml.safe_load(recipe_path.read_text())["resources"] - if "agg_nodes" in resources: - return int(resources["agg_nodes"]) - if "prefill_nodes" in resources and "decode_nodes" in resources: - return int(resources["prefill_nodes"]) + int(resources["decode_nodes"]) - raise ValueError(f"Recipe has no supported node resource fields: {recipe_path}") + recipe = yaml.safe_load(recipe_path.read_text()) + if recipe.get("schema") != 2: + raise ValueError(f"srt-slurm recipes must declare schema: 2: {recipe_path}") + if "base" in recipe: + # A file with several override variants has no single authoritative + # node count. The selected master topology supplies the estimate. + return None + roles = recipe.get("roles") + if roles: + # Schema 2 groups node allocations by role. A colocated decode role + # shares prefill nodes and does not reserve another allocation. + for name, role in roles.items(): + if "nodes" not in role: + raise ValueError( + f"Recipe role {name!r} must specify nodes: {recipe_path}" + ) + return sum( + 0 if role["nodes"] == "colocate" else int(role["nodes"]) + for role in roles.values() + ) + raise ValueError(f"Recipe has no worker roles: {recipe_path}") def worker_node_count( diff --git a/infx/results/power/multinode.py b/infx/results/power/multinode.py index 7899b067d3..cbdf2992c2 100644 --- a/infx/results/power/multinode.py +++ b/infx/results/power/multinode.py @@ -82,6 +82,9 @@ "power_w", ) +# srt-slurm v2 appends optional utilization fields to the power samples. +SAMPLES_HEADER_V2 = (*SAMPLES_HEADER, "gpu_util_pct", "sm_active") + # Fixed by the producer contract (srt-slurm contract.MAX_SAMPLE_GAP_SECONDS), # NOT a multiple of the configured sample interval. MAX_SAMPLE_GAP_SECONDS = 3.0 @@ -331,8 +334,10 @@ def _check_wire_contract(manifest: dict) -> list[str]: # --- strict samples parsing (mirrors srt-slurm samples.read_samples) -------- -def _parse_sample_row(raw: list[str]) -> SampleRow | None: - if len(raw) != len(SAMPLES_HEADER): +def _parse_sample_row(raw: list[str], expected_version: int) -> SampleRow | None: + """Validate the selected CSV generation, including optional utilization.""" + header = SAMPLES_HEADER_V2 if expected_version == 2 else SAMPLES_HEADER + if len(raw) != len(header): return None try: schema_version = int(raw[0]) @@ -340,10 +345,16 @@ def _parse_sample_row(raw: list[str]) -> SampleRow | None: scrape_seq = int(raw[2]) gpu_index = int(raw[4]) power_w = float(raw[6]) + if expected_version == 2: + for cell, maximum in zip(raw[7:], (100.0, 1.0)): + if cell: + value = float(cell) + if not math.isfinite(value) or not 0 <= value <= maximum: + return None except ValueError: return None hostname, gpu_uuid = raw[3], raw[5] - if schema_version != SCHEMA_VERSION or not hostname or not gpu_uuid: + if schema_version != expected_version or not hostname or not gpu_uuid: return None if not math.isfinite(timestamp_unix) or not math.isfinite(power_w) or power_w < 0: return None @@ -369,10 +380,14 @@ def read_samples(path: Path) -> tuple[tuple[SampleRow, ...], tuple[str, ...]]: with open(path, newline="", encoding="utf-8") as handle: reader = csv.reader(handle) header = next(reader, None) - if header != list(SAMPLES_HEADER): + if header == list(SAMPLES_HEADER): + expected_version = 1 + elif header == list(SAMPLES_HEADER_V2): + expected_version = 2 + else: return (), ("samples_csv_header_mismatch",) for raw in reader: - row = _parse_sample_row(raw) + row = _parse_sample_row(raw, expected_version) if row is None: reasons.append("samples_csv_malformed") continue diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index 1465ab483c..e8b9c948e2 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -1,14 +1,12 @@ #!/usr/bin/bash # shellcheck source=runners/slurm_utils.sh -source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" +source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" || exit 1 # Compatibility launcher for B200 Nscale configurations that have not yet # moved to the native srt-slurm path in launch_b200-nscale-slurm.sh. SLURM_PARTITION="${SLURM_PARTITION:-batch_1}" SLURM_ACCOUNT="${SLURM_ACCOUNT:-benchmark}" -POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" -POWER_SRT_SLURM_PIN="e5c837f06a362dc888dfea2ee588e9f19c298270" set -x @@ -145,7 +143,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ -n "$_POWER_CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' /^telemetry:/ { t = 1; next } t && /^[^ ]/ { t = 0 } - t && /^ provider: dcgm-power$/ { p = 1 } + t && /^ dcgm_exporter:/ { p = 1 } t && /^ enabled: true$/ { e = 1 } END { exit !(p && e) } ' "$_RECIPE_SRC"; then @@ -163,79 +161,14 @@ if [[ "$IS_MULTINODE" == "true" ]]; then export SERVED_MODEL_NAME=$MODEL - echo "Cloning srt-slurm repository..." + echo "Preparing job-local srt-slurm checkout..." SRT_REPO_DIR="srt-slurm" if [ -d "$SRT_REPO_DIR" ]; then echo "Removing existing $SRT_REPO_DIR..." rm -rf "$SRT_REPO_DIR" fi - # Kimi K3 aggregate profiles use the srt-slurm fork that supports direct - # multi-node vLLM. Pin the tested renderer so branch movement cannot change - # generated rank commands between sweep points. - if [[ "$USES_DCGM_POWER" == "1" ]]; then - git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - git checkout "$POWER_SRT_SLURM_PIN" || exit 1 - test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } - git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" - mkdir -p recipes/vllm/deepseek-v4 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 - elif [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then - git clone --branch klaud/direct-vllm-multinode --single-branch https://github.com/functionstackx/srt-slurm-nv.git "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - git checkout df5baa93f4caf5169dea2a4236ad2cc742fe40e7 || exit 1 - mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ - recipes/vllm/kimi-k3/agentic || exit 1 - elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" || exit 1 - git checkout aflowers/vllm-gb200-v0.20.0 - mkdir -p recipes/vllm/deepseek-v4 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 - elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then - git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" || exit 1 - git checkout c180328b98c3793ca84a1e24a030f90545eb7d5d || exit 1 - mkdir -p recipes/vllm/kimi-k2.6 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6" recipes/vllm/kimi-k2.6 - elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" || exit 1 - mkdir -p recipes/vllm/minimax-m3/b200-fp4 - cp -rT \ - "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4" \ - recipes/vllm/minimax-m3/b200-fp4 - elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" || exit 1 - git checkout main - elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" || exit 1 - # Pin srt-slurm: newer commits stopped honoring the hash-pinned dynamo - # build and fall back to a dynamo release that is incompatible with this - # sglang image (worker fails at import). This is the last commit before - # that change. Do not float on main -- the srtctl + dynamo-install - # toolchain is unpinned there. - git checkout a98738de9b2233459b5456e9ed71af09ce893f92 - mkdir -p recipes/sglang/dsr1/b200-fp4 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4" recipes/sglang/dsr1/b200-fp4 - elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" || exit 1 - git checkout v1.0.29 - mkdir -p recipes/trtllm/kimi-k25-nvfp4/b200-fp4 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4" recipes/trtllm/kimi-k25-nvfp4/b200-fp4 - else - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" || exit 1 - git checkout sa-submission-q2-2026 - fi - if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then - python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" || exit 1 - fi + setup_srt_slurm "$SRT_REPO_DIR" "$FRAMEWORK" "$USES_DCGM_POWER" || exit 1 echo "Installing srtctl..." export UV_INSTALL_DIR="$GITHUB_WORKSPACE/.local/bin" @@ -411,7 +344,7 @@ EOF SRTCTL_PREFLIGHT_ARGS+=(--no-preflight) fi - SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" "${SRTCTL_PREFLIGHT_ARGS[@]}" --tags "b200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) + SRTCTL_OUTPUT=$(srtctl apply "${SRTCTL_EVAL_ARGS[@]}" -f "$CONFIG_FILE" "${SRTCTL_PREFLIGHT_ARGS[@]}" --tags "b200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) echo "$SRTCTL_OUTPUT" # Extract JOB_ID from srtctl output diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index 474fe81ac7..a6b448634c 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -11,11 +11,6 @@ SLURM_PARTITION="batch_1" SLURM_ACCOUNT="benchmark" -POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" -POWER_SRT_SLURM_PIN="e5c837f06a362dc888dfea2ee588e9f19c298270" -AGENTX_POWER_SRT_SLURM_PIN="80d7203e424f903c9017de4608ee2044afce9574" -TILERT_SRT_SLURM_URL="https://github.com/SemiAnalysisAI/srt-slurm.git" -TILERT_SRT_SLURM_PIN="d1e6c97b3baf3e87103b6d83189544c3c7d61c38" # Node-local NVMe, not a shared filesystem: much faster for the ~1.6T # DeepSeek-V4-Pro load, and already pre-staged on every nscale compute node. @@ -27,7 +22,7 @@ HF_HUB_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/hf-hub-cache" SQUASH_LOCK_TIMEOUT=3600 # shellcheck source=runners/slurm_utils.sh -source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" +source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" || exit 1 set -x @@ -83,7 +78,7 @@ _RECIPE_SRC="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${_RECIPE if [[ -n "$_POWER_CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' /^telemetry:/ { t = 1; next } t && /^[^ ]/ { t = 0 } - t && /^ provider: dcgm-power$/ { p = 1 } + t && /^ dcgm_exporter:/ { p = 1 } t && /^ enabled: true$/ { e = 1 } END { exit !(p && e) } ' "$_RECIPE_SRC"; then @@ -105,90 +100,10 @@ fi export SERVED_MODEL_NAME=$MODEL -echo "Cloning srt-slurm repository..." +echo "Preparing job-local srt-slurm checkout..." SRT_REPO_DIR="srt-slurm" rm -rf "$SRT_REPO_DIR" -if [[ "$USES_DCGM_POWER" == "1" ]]; then - SELECTED_POWER_SRT_SLURM_PIN="$POWER_SRT_SLURM_PIN" - if [[ "$USES_AGENTX_POWER" == "1" ]]; then - SELECTED_POWER_SRT_SLURM_PIN="$AGENTX_POWER_SRT_SLURM_PIN" - fi - git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - git checkout "$SELECTED_POWER_SRT_SLURM_PIN" || exit 1 - test "$(git rev-parse HEAD)" = "$SELECTED_POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match selected power producer $SELECTED_POWER_SRT_SLURM_PIN" >&2; exit 1; } - git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" - if [[ "$USES_AGENTX_POWER" == "1" ]]; then - mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ - recipes/vllm/kimi-k3/agentic || exit 1 - elif [[ "$MODEL_PREFIX" == "dsv4" && "$FRAMEWORK" == "dynamo-sglang" ]]; then - mkdir -p recipes/sglang/deepseek-v4 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 - elif [[ "$MODEL_PREFIX" == "dsv4" ]]; then - mkdir -p recipes/vllm/deepseek-v4 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 - else - mkdir -p recipes/vllm/kimi-k2.6 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6" recipes/vllm/kimi-k2.6 - fi -elif [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "glm5.1" && $FRAMEWORK == "tilert" ]]; then - git clone "$TILERT_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - git checkout "$TILERT_SRT_SLURM_PIN" || exit 1 - test "$(git rev-parse HEAD)" = "$TILERT_SRT_SLURM_PIN" || { - echo "Error: srt-slurm HEAD does not match TILERT_SRT_SLURM_PIN=$TILERT_SRT_SLURM_PIN" >&2 - exit 1 - } - mkdir -p recipes/tilert/glm5.1/b200-fp8/agentic || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/tilert/glm5.1/b200-fp8/agentic" \ - recipes/tilert/glm5.1/b200-fp8/agentic || exit 1 -elif [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then - # Pin the tested renderer so branch movement cannot change generated rank - # commands between sweep points. - git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - git checkout 217f9438 || exit 1 - mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ - recipes/vllm/kimi-k3/agentic || exit 1 -elif [[ $MODEL_PREFIX == "glm5.2" && $FRAMEWORK == "dynamo-sglang" ]]; then - # Pin the renderer used for the validated checked-in recipes. - git clone --branch v1.0.53 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - test "$(git rev-parse HEAD)" = "217f94387abeddfed7149a71955dc523e07cd765" || { - echo "Error: srt-slurm v1.0.53 did not resolve to the pinned commit" >&2 - exit 1 - } - mkdir -p recipes/sglang/glm5.2/b200-fp4/agentic || exit 1 - cp -rT \ - "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/b200-fp4/agentic" \ - recipes/sglang/glm5.2/b200-fp4/agentic || exit 1 -elif [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK == "dynamo-sglang" ]]; then - git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - # Pin the srt-slurm revision used by these checked-in recipes. - git checkout 04e87fcc505d6d851451781a5499ca19a02ec2b4 || exit 1 - mkdir -p recipes/sglang/deepseek-v4 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 -elif [[ $MODEL_PREFIX == "dsv4" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - git checkout aflowers/vllm-gb200-v0.20.0 || exit 1 - mkdir -p recipes/vllm/deepseek-v4 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 -else - git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - git checkout c180328b98c3793ca84a1e24a030f90545eb7d5d || exit 1 - mkdir -p recipes/vllm/kimi-k2.6 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6" recipes/vllm/kimi-k2.6 -fi -if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then - python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" \ - || exit 1 -fi - +setup_srt_slurm "$SRT_REPO_DIR" "$FRAMEWORK" "$USES_DCGM_POWER" || exit 1 echo "Installing srtctl..." export UV_INSTALL_DIR="$GITHUB_WORKSPACE/.local/bin" @@ -397,7 +312,7 @@ if [[ $MODEL_PREFIX == "kimik2.6" ]] || SRTCTL_PREFLIGHT_ARGS+=(--no-preflight) fi -SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" "${SRTCTL_PREFLIGHT_ARGS[@]}" --tags "b200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) +SRTCTL_OUTPUT=$(srtctl apply "${SRTCTL_EVAL_ARGS[@]}" -f "$CONFIG_FILE" "${SRTCTL_PREFLIGHT_ARGS[@]}" --tags "b200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) echo "$SRTCTL_OUTPUT" JOB_ID=$(echo "$SRTCTL_OUTPUT" | grep -oP '✅ Job \K[0-9]+' || echo "$SRTCTL_OUTPUT" | grep -oP 'Job \K[0-9]+') @@ -437,7 +352,7 @@ if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" \ "$GITHUB_WORKSPACE" "$GITHUB_WORKSPACE" "$RESULT_FILENAME" \ - "$SELECTED_POWER_SRT_SLURM_PIN" "${POWER_CONCURRENCIES[@]}" || AGENTX_POWER_RC=$? + "$SRT_SLURM_COMMIT" "${POWER_CONCURRENCIES[@]}" || AGENTX_POWER_RC=$? fi if [[ "$USES_DCGM_POWER" == "1" ]]; then diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index 0f13a1721c..a72fe27b21 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -1,7 +1,7 @@ #!/usr/bin/bash # shellcheck source=runners/slurm_utils.sh -source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" +source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" || exit 1 # Launcher for the B300 DSXE Slurm cluster (dsxe-sa-b300-prd0), runners run as sa-gha-runner. # @@ -22,10 +22,6 @@ MODEL_ROOT="/scratch/models" SHARED_MODEL_ROOT="/data/models" WRITABLE_MODELS_DIR="/data/home/sa-gha-runner/models" -# Official power (dcgm-power) runs use a separate, pinned producer; CI derives -# POWER_PRODUCER_SHA from the stamp this script writes. Keep in sync with the other launchers. -POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" -POWER_SRT_SLURM_PIN="e5c837f06a362dc888dfea2ee588e9f19c298270" # Directory names under MODEL_ROOT (upstream HF repo basenames). STAGED_MODELS=( @@ -127,7 +123,7 @@ _RECIPE_SRC="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${_RECIPE if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' /^telemetry:/ { t = 1; next } t && /^[^ ]/ { t = 0 } - t && /^ provider: dcgm-power$/ { p = 1 } + t && /^ dcgm_exporter:/ { p = 1 } t && /^ enabled: true$/ { e = 1 } END { exit !(p && e) } ' "$_RECIPE_SRC"; then @@ -143,48 +139,9 @@ if [[ "$USES_DCGM_POWER" == "1" && ( exit 1 fi -# Default is the newest tag. Add a branch here to pin a ref per model / precision / -# framework when a recipe needs one, so results stay reproducible. -select_srt_slurm_version() { - if false; then - : - else - SRT_SLURM_REPO="https://github.com/NVIDIA/srt-slurm.git" - SRT_SLURM_REF="v1.0.87" - fi -} - -# --------------------------------------------------------------------------- -# srt-slurm checkout: one clone at the selected ref, plus every in-repo recipe. -# --------------------------------------------------------------------------- SRT_REPO_DIR="srt-slurm" rm -rf "$SRT_REPO_DIR" - -if [[ "$USES_DCGM_POWER" == "1" ]]; then - SRT_SLURM_REPO="$POWER_SRT_SLURM_URL" - SRT_SLURM_REF="$POWER_SRT_SLURM_PIN" -else - select_srt_slurm_version -fi - -echo "Cloning srt-slurm ($SRT_SLURM_REPO @ $SRT_SLURM_REF)..." -git clone "$SRT_SLURM_REPO" "$SRT_REPO_DIR" || exit 1 -cd "$SRT_REPO_DIR" || exit 1 -git checkout --quiet "$SRT_SLURM_REF" || exit 1 -git rev-parse HEAD > "$GITHUB_WORKSPACE/srt-slurm-sha.txt" -if [[ "$USES_DCGM_POWER" == "1" ]]; then - test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" \ - || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } - cp "$GITHUB_WORKSPACE/srt-slurm-sha.txt" "$GITHUB_WORKSPACE/power-producer-sha.txt" -fi - -# Recipes live in this repo; overlay all of them onto the checkout's recipes/ dir. -mkdir -p recipes -cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes" recipes || exit 1 - -if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then - python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" || exit 1 -fi +setup_srt_slurm "$SRT_REPO_DIR" "$FRAMEWORK" "$USES_DCGM_POWER" || exit 1 echo "Installing srtctl..." export UV_INSTALL_DIR="$GITHUB_WORKSPACE/.local/bin" @@ -294,7 +251,7 @@ SRTCTL_APPLY_ARGS=( --no-preflight --tags "b300,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" ) -SRTCTL_OUTPUT=$(srtctl apply "${SRTCTL_APPLY_ARGS[@]}" 2>&1) +SRTCTL_OUTPUT=$(srtctl apply "${SRTCTL_EVAL_ARGS[@]}" "${SRTCTL_APPLY_ARGS[@]}" 2>&1) echo "$SRTCTL_OUTPUT" # Extract JOB_ID from srtctl output diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 8134938452..924dee71cd 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -4,19 +4,12 @@ set -x -source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" +source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" || exit 1 export SLURM_PARTITION="batch" export SLURM_ACCOUNT="benchmark" SQUASH_DIR="/mnt/lustre01/users-public/sa-shared" -# Fixed-sequence dcgm-power producer pin. Swap -# URL+PIN here (and identically in launch_gb300-nv.sh) when the upstream -# srt-slurm merge lands. -POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" -POWER_SRT_SLURM_PIN="6fc1bed01a0b82dae0088a105c03ce0cfb353443" -AGENTX_POWER_SRT_SLURM_PIN="80d7203e424f903c9017de4608ee2044afce9574" - # Enroot 3.x does not parse Docker's tag@digest syntax. For digest-pinned # images, use its explicit registry syntax and pass the digest as the # manifest reference so the import remains immutable. @@ -319,7 +312,7 @@ import_squash "$NGINX_SQUASH_FILE" "$NGINX_IMAGE" # Power lane is recipe-driven: on iff the recipe this run resolves carries an # enabled dcgm-power telemetry block. Read the workspace mirror (it overlays -# the srt-slurm clone later), since the pin decision precedes the clone. +# the srt-slurm clone later). USES_DCGM_POWER=0 _RECIPE_REL="${CONFIG_FILE%%:*}" _RECIPE_SRC="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${_RECIPE_REL#recipes/}" @@ -328,17 +321,14 @@ _RECIPE_SRC="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${_RECIPE if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' /^telemetry:/ { t = 1; next } t && /^[^ ]/ { t = 0 } - t && /^ provider: dcgm-power$/ { p = 1 } + t && /^ dcgm_exporter:/ { p = 1 } t && /^ enabled: true$/ { e = 1 } END { exit !(p && e) } ' "$_RECIPE_SRC"; then USES_DCGM_POWER=1 fi -# Note (wenyao): the producer pin follows the srt-slurm main lineage that the -# dynamo-sglang lanes run on (fp8 validated end-to-end, fp4 recipes -# parse-verified against the pin); other frameworks clone diverging refs -# (aflowers branch, sa-submission), so fail fast for them instead. +# Keep the existing framework allowlist for official power validation. if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" ]]; then echo "Error: dcgm-power lanes are only validated for FRAMEWORK=dynamo-sglang, got: $FRAMEWORK" >&2 exit 1 @@ -347,7 +337,7 @@ fi USES_AGENTX_POWER=0 if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" ]]; then if [[ "$MODEL_PREFIX" == "glm5.2" && "$PRECISION" == "fp4" && - "$_RECIPE_REL" == "recipes/sglang/glm5.2/gb200-fp4/agentic/glm5.2-agentx-agg.yaml" ]]; then + "$_RECIPE_REL" == "recipes/glm5.2/sglang/gb200-fp4/agentx/agg.yaml" ]]; then USES_AGENTX_POWER=1 else echo "Error: GB200 AgentX dcgm-power requires the GLM-5.2 aggregate recipe" >&2 @@ -429,9 +419,8 @@ if [[ -z "$CONFIG_FILE" ]]; then exit 1 fi -echo "Cloning srt-slurm repository..." +echo "Preparing job-local srt-slurm checkout..." SRT_REPO_DIR="srt-slurm" -SRTCTL_SETUP_SCRIPT="" if uses_watchtower_shared_fs; then SHARED_BASE="/mnt/lustre01/users-public/sa-shared/gha-runs" RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RUNNER_NAME}-$$" @@ -442,188 +431,11 @@ if [ -d "$SRT_REPO_DIR" ]; then rm -rf "$SRT_REPO_DIR" fi -# AgentX power needs the custom-window producer contract; the released GLM -# metrics path can keep its existing producer. -if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "glm5.2" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-sglang" ]]; then - if [[ "$USES_AGENTX_POWER" == "1" ]]; then - git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - git checkout "$AGENTX_POWER_SRT_SLURM_PIN" || exit 1 - test "$(git rev-parse HEAD)" = "$AGENTX_POWER_SRT_SLURM_PIN" || { - echo "Error: srt-slurm HEAD does not match AgentX power producer $AGENTX_POWER_SRT_SLURM_PIN" >&2 - exit 1 - } - git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" - else - git clone --branch v1.0.50 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - test "$(git rev-parse HEAD)" = "e4019633c9e2bc25f38c44b81edf52bb0504d937" || { - echo "Error: NVIDIA/srt-slurm v1.0.50 resolved to an unexpected commit" >&2 - exit 1 - } - fi - mkdir -p recipes/sglang/glm5.2/gb200-fp4/agentic - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb200-fp4/agentic" \ - recipes/sglang/glm5.2/gb200-fp4/agentic -elif [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "minimaxm3" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-vllm" ]]; then - SRT_SLURM_MINIMAX_PIN="d50ee7280c33d469df8708e363e23be2456e94fb" - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout "$SRT_SLURM_MINIMAX_PIN" - test "$(git rev-parse HEAD)" = "$SRT_SLURM_MINIMAX_PIN" || { - echo "Error: NVIDIA/srt-slurm MiniMax-M3 revision resolved to an unexpected commit" >&2 - exit 1 - } - mkdir -p recipes/vllm/minimax-m3/gb200-fp4/agentic - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic" \ - recipes/vllm/minimax-m3/gb200-fp4/agentic -# These AgentX submissions use released srt-slurm custom-benchmark metrics -# discovery so AIPerf receives every logical worker endpoint. -elif [[ "$IS_AGENTIC" == "1" && (( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-sglang" ) || ( "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-vllm" )) ]]; then - git clone --branch v1.0.45 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - test "$(git rev-parse HEAD)" = "9d8d92b20c350a5d42f0709f5a0b64e30eb37d33" || { - echo "Error: NVIDIA/srt-slurm v1.0.45 resolved to an unexpected commit" >&2 - exit 1 - } - if [[ "$MODEL_PREFIX" == "qwen3.5" ]]; then - mkdir -p recipes/sglang/qwen3.5/gb200-fp4/agentic - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic" \ - recipes/sglang/qwen3.5/gb200-fp4/agentic - else - mkdir -p recipes/vllm/deepseek-v4/agentic - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ - recipes/vllm/deepseek-v4/agentic - fi -# Kimi-K3 requires direct multi-node vLLM frontend support from srt-slurm. -elif [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "kimik3" ]]; then - git clone --branch v1.0.53 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - test "$(git rev-parse HEAD)" = "217f94387abeddfed7149a71955dc523e07cd765" || { - echo "Error: NVIDIA/srt-slurm v1.0.53 resolved to an unexpected commit" >&2 - exit 1 - } - python3 "$GITHUB_WORKSPACE/runners/patch_srt_vllm_dp_ranks.py" "$(pwd)" || exit 1 - mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ - recipes/vllm/kimi-k3/agentic || exit 1 -elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "minimaxm3" ]]; then - # Select this before the generic Agentic fallback. v1.0.91 carries - # srt-slurm #384 (numactl -m 0,1 on aggregated TRT-LLM workers). - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - git checkout v1.0.91 || exit 1 - # Preserve the repository-relative CONFIG_FILE paths inside this checkout. - RECIPE_DIR="benchmarks/multi_node/srt-slurm-recipes/trtllm/minimax-m3/gb200-fp4/agentic" - mkdir -p "$RECIPE_DIR" || exit 1 - cp -rT "$GITHUB_WORKSPACE/$RECIPE_DIR" "$RECIPE_DIR" || exit 1 -# TODO(CJQ): migrate the remaining Agentic model paths to released srt-slurm. -elif [[ "$IS_AGENTIC" == "1" ]]; then - # Agentic multi-node pins cquil11/srt-slurm-nv revisions that provide: - # - BenchmarkType.CUSTOM + benchmark.command + benchmark.env - # (the hook that hands off to benchmarks/multi_node/agentic_srt.sh) - # - DynamoConfig.wheel (recipes pin the ai-dynamo wheel) - # - srtctl apply --no-preflight (model path /mnt/numa1 is compute-node - # local NVMe, invisible to the login-node runner) - # - benchmark_stage srun_options propagation (container-remap-root - # must reach the agentic_srt.sh srun) - git clone https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout de59739b172e507e15ebf145bfe305f606e82fbf - mkdir -p recipes/vllm/deepseek-v4/agentic - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ - recipes/vllm/deepseek-v4/agentic -elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout v1.0.31 - # Use `cp -rT` so if the upstream branch ever ships a stub - # `recipes/vllm/deepseek-v4/` directory, we overlay our recipes onto - # it rather than nesting (`cp -r src dst` would create - # `recipes/vllm/deepseek-v4/deepseek-v4/...` in that case). - mkdir -p recipes/vllm/deepseek-v4 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 -elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then - if [[ "$USES_DCGM_POWER" == "1" ]]; then - # Note (wenyao): on this cluster the DSV4-Pro checkpoint lives on the - # compute-node /mnt/numa1 NVMe (same staging the agentic path and the - # llm-d sweeps load from); the lustre alias target the shared dsv4 - # block exports is not present here. Scoped to the power lane so the - # non-power lane keeps whatever the external-cluster staging expects. - export MODEL_PATH="/mnt/numa1/models/DeepSeek-V4-Pro" - git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout "$POWER_SRT_SLURM_PIN" || exit 1 - # The power lane must run the exact pinned producer SHA, never a moving branch. - test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } - git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" - else - # Stay on NVIDIA/srt-slurm:main (default) — submission branch no - # longer needed; overlay our hand-rolled DSV4 sglang recipes onto it. - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - fi - mkdir -p recipes/sglang/deepseek-v4 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 -elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.1" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout sa-submission-q2-2026 - mkdir -p recipes/sglang/glm5 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5" recipes/sglang/glm5 -elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then - if [[ "$USES_DCGM_POWER" == "1" ]]; then - # Power lanes run the exact pinned producer SHA, never a moving - # branch; CI derives POWER_PRODUCER_SHA from the stamp file. - git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout "$POWER_SRT_SLURM_PIN" || exit 1 - test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } - git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" - else - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - fi - mkdir -p recipes/sglang/qwen3.5 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" recipes/sglang/qwen3.5 -elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.1" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - mkdir -p recipes/sglang/glm5 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5" recipes/sglang/glm5 -elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - git checkout sa-submission-q2-2026 || exit 1 - mkdir -p recipes/vllm/minimax-m3-gb200-fp8 || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8" recipes/vllm/minimax-m3-gb200-fp8 || exit 1 -elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - git checkout main || exit 1 - mkdir -p recipes/vllm/kimi-k2.5-fp4 || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4" recipes/vllm/kimi-k2.5-fp4 || exit 1 -elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout sa-submission-q2-2026 -elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "kimik2.5" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout sa-submission-q2-2026 -elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "glm5" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout v1.0.26 - mkdir -p recipes/trtllm/glm5 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5" recipes/trtllm/glm5 -else - git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" -fi -if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then - python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" || exit 1 +# This checkpoint is staged on compute-node NVMe for the power lane. +if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" == "dynamo-sglang" && "$MODEL_PREFIX" == "dsv4" && "$IS_AGENTIC" != "1" ]]; then + export MODEL_PATH="/mnt/numa1/models/DeepSeek-V4-Pro" fi +setup_srt_slurm "$SRT_REPO_DIR" "$FRAMEWORK" "$USES_DCGM_POWER" || exit 1 echo "Installing srtctl..." curl -LsSf https://astral.sh/uv/install.sh | sh @@ -816,13 +628,11 @@ SRTCTL_APPLY_ARGS=( ) if [[ "$FRAMEWORK" == "dynamo-sglang" ]]; then SRTCTL_APPLY_ARGS+=(--setup-script install-torchao.sh) -elif [[ -n "$SRTCTL_SETUP_SCRIPT" ]]; then - SRTCTL_APPLY_ARGS+=(--setup-script "$SRTCTL_SETUP_SCRIPT") fi # srtctl gives the GitHub-provided RUNNER_NAME precedence over config.name. # Override it only for submission so the rendered #SBATCH job name retains # the InferenceX namespace used above. -SRTCTL_OUTPUT=$(RUNNER_NAME="$SRT_SLURM_JOB_NAME" srtctl apply "${SRTCTL_APPLY_ARGS[@]}" 2>&1) +SRTCTL_OUTPUT=$(RUNNER_NAME="$SRT_SLURM_JOB_NAME" srtctl apply "${SRTCTL_EVAL_ARGS[@]}" "${SRTCTL_APPLY_ARGS[@]}" 2>&1) echo "$SRTCTL_OUTPUT" JOB_ID=$(echo "$SRTCTL_OUTPUT" | grep -oP '✅ Job \K[0-9]+' || echo "$SRTCTL_OUTPUT" | grep -oP 'Job \K[0-9]+') @@ -897,7 +707,7 @@ if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then --agg-result "$GITHUB_WORKSPACE/${RESULT_FILENAME}_conc${concurrency}.json" \ --power-dir "$POWER_LOGS_ROOT/power" \ --logs-root "$POWER_LOGS_ROOT" \ - --expected-producer-sha "$AGENTX_POWER_SRT_SLURM_PIN" \ + --expected-producer-sha "$SRT_SLURM_COMMIT" \ --require-power ) || AGENTX_POWER_RC=$? done diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 0b3916c71c..cac6747b58 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -5,7 +5,7 @@ set -exo pipefail # shellcheck source=runners/slurm_utils.sh -source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" +source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" || exit 1 export SLURM_PARTITION="${SLURM_PARTITION:-batch_1}" export SBATCH_PARTITION="$SLURM_PARTITION" @@ -188,7 +188,7 @@ _RECIPE_SRC="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${_RECIPE if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' /^telemetry:/ { t = 1; next } t && /^[^ ]/ { t = 0 } - t && /^ provider: dcgm-power$/ { p = 1 } + t && /^ dcgm_exporter:/ { p = 1 } t && /^ enabled: true$/ { e = 1 } END { exit !(p && e) } ' "$_RECIPE_SRC"; then @@ -199,7 +199,7 @@ USES_AGENTX_POWER=0 if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "kimik3" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-vllm" && - "$_RECIPE_REL" == recipes/vllm/kimi-k3/agentic/* ]]; then + "$_RECIPE_REL" == recipes/kimik3/vllm/*/agentx/* ]]; then USES_AGENTX_POWER=1 fi if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" && "$USES_AGENTX_POWER" != "1" ]]; then @@ -207,12 +207,6 @@ if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" && "$USES_AGE exit 1 fi -# dcgm-power producer pin — single source of truth for power lanes. Swap -# URL+PIN here (and identically in launch_gb200-nv.sh) when the upstream -# srt-slurm merge lands. -POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" -POWER_SRT_SLURM_PIN="6fc1bed01a0b82dae0088a105c03ce0cfb353443" -AGENTX_POWER_SRT_SLURM_PIN="80d7203e424f903c9017de4608ee2044afce9574" if [[ "$USES_DCGM_POWER" == "1" ]]; then DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" @@ -238,250 +232,26 @@ fi export ISL="$ISL" export OSL="$OSL" -echo "Cloning srt-slurm repository..." +echo "Preparing job-local srt-slurm checkout..." RUN_KEY=$(printf "%s" "${RESULT_FILENAME:-${RUNNER_NAME:-gb300-nv}}" | sha1sum | cut -c1-12) SRT_REPO_DIR="${GITHUB_WORKSPACE}/srt-slurm-${GITHUB_RUN_ID:-manual}-${GITHUB_RUN_ATTEMPT:-0}-${RUN_KEY}" -SRTCTL_SETUP_SCRIPT="" rm -rf "$SRT_REPO_DIR" -if [[ "$USES_AGENTX_POWER" == "1" ]]; then - git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - git checkout "$AGENTX_POWER_SRT_SLURM_PIN" || exit 1 - test "$(git rev-parse HEAD)" = "$AGENTX_POWER_SRT_SLURM_PIN" || exit 1 - git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" - mkdir -p recipes/vllm/kimi-k3/agentic - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ - recipes/vllm/kimi-k3/agentic -elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qwen3.5" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout v1.0.50 - TRTLLM_RECIPES_DIR="recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx" - mkdir -p "$TRTLLM_RECIPES_DIR" - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx" \ - "$TRTLLM_RECIPES_DIR" - if [[ "${EVAL_ONLY:-false}" == "true" ]]; then - find "$TRTLLM_RECIPES_DIR" -name "*.yaml" \ - -exec sed -i '/TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS/d' {} + - fi -elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "glm5.2" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - # The GB300 AgentX recipes put the benchmark client on its own allocation node - # (client_dedicated_node), a schema added in srt-slurm v1.0.62; v1.0.72 is the - # release this runner's Dynamo TRT path already exercises. - git checkout v1.0.72 - TRTLLM_RECIPES_DIR="benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2" - mkdir -p "$TRTLLM_RECIPES_DIR" - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2" \ - "$TRTLLM_RECIPES_DIR" - if [[ "${EVAL_ONLY:-false}" == "true" ]]; then - find "$TRTLLM_RECIPES_DIR" -name "*.yaml" \ - -exec sed -i '/TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS/d' {} + - fi -elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then - # Qwen3.5 agentic uses NVIDIA/srt-slurm v1.0.38: the two features the - # cquil11 fork was pinned for are merged upstream (present in v1.0.36) — - # - `srtctl apply --no-preflight` (skip the in-process model FS check): - # model.path resolves to /scratch/models/Qwen3.5-397B-A17B-NVFP4 - # (compute-node-only NVMe), which the GHA runner pod can't stat, so - # the Path.is_dir() preflight would fail before sbatch is ever - # called. The engine still fails loudly at runtime if the path is - # genuinely missing on the compute node. - # - benchmark_stage propagates srun_options (container-remap-root must - # reach the agentic_srt.sh srun). - # v1.0.38 additionally injects AIPERF_SERVER_METRICS_URLS for custom - # benchmarks using each logical SGLang worker leader. This is required for - # complete AgentX trace artifacts; the public frontend alone may expose no - # Prometheus endpoint or only Dynamo frontend metrics. - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout v1.0.38 - mkdir -p recipes/sglang/qwen3.5 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" \ - recipes/sglang/qwen3.5 -elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then - # DSv4 GB300 SGLang agentic uses NVIDIA/srt-slurm v1.0.40. In addition to - # the nginx body-size fix, session-affinity frontend, and custom benchmark - # schema required by these recipes, this release injects every logical - # SGLang worker leader's /metrics URL into AIPERF_SERVER_METRICS_URLS and - # supports long-lived nginx keepalive for multi-turn AgentX replay. - # AgentX forwards that list to aiperf's --server-metrics argument so its - # trace artifacts include backend metrics for every engine. - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout v1.0.40 - mkdir -p recipes/sglang/deepseek-v4/agentic - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic" \ - recipes/sglang/deepseek-v4/agentic -elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "dsv4" ]]; then - SRT_SLURM_MODEL_PREFIX="deepseek-ai/DeepSeek-V4-Pro" - git clone --branch v1.0.50 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - - mkdir -p benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4 || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4" \ - benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4 || exit 1 - if [[ "${EVAL_ONLY:-false}" == "true" ]]; then - # srt-slurm v1.0.50 launches lm-eval on the allocation head and uses - # localhost:8000. Keep AgentX frontends on first_decode for throughput, - # but co-locate the eval-only frontend with lm-eval so loopback resolves. - find benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4 -name "*.yaml" \ - -exec sed -i \ - -e '/TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS/d' \ - -e 's/^ orchestrator_placement: first_decode$/ orchestrator_placement: head/' \ - {} + - fi -elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.2" ]]; then - # GLM-5.2 GB300 sglang AgentX: srt-slurm main has the agentx-mvp scenario, - # the zip_override sweep selectors, and the multi-frontend session-affinity - # schema these recipes need. - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout main - mkdir -p recipes/sglang/glm5.2/gb300-fp4 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb300-fp4" \ - recipes/sglang/glm5.2/gb300-fp4 -elif [[ "$IS_AGENTIC" == "1" ]]; then - # Agentic recipes use NVIDIA/srt-slurm v1.0.36. This is the upstream - # version validated in InferenceX PR #2302 and includes per-node DP, - # matching Dynamo health counts, multi-node TP port handling, and - # Mooncake compatibility. Keep it pinned so sweeps are reproducible. - git clone --branch v1.0.36 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - - mkdir -p recipes/vllm/deepseek-v4/agentic || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ - recipes/vllm/deepseek-v4/agentic || exit 1 - mkdir -p recipes/vllm/minimax-m3/agentic || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/agentic" \ - recipes/vllm/minimax-m3/agentic || exit 1 - mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ - recipes/vllm/kimi-k3/agentic || exit 1 -elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout aflowers/gb200-dsv4-recipes - mkdir -p recipes/vllm/deepseek-v4 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 -elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then - # Fixed-length DeepSeek-V4 recipes are version-controlled in this repository; - # overlay them onto the selected srt-slurm checkout. Power lanes use the - # exact producer pin and stamp it for strict result provenance; non-power - # lanes retain the v1.0.25 release that bootstraps cargo/maturin for the - # hash-pinned Dynamo source build before launch. - if [[ "$USES_DCGM_POWER" == "1" ]]; then - git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - git checkout "$POWER_SRT_SLURM_PIN" || exit 1 - # The power lane must run the exact pinned producer SHA, never a moving branch. - test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } - git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" - else - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - git checkout v1.0.25 || exit 1 - fi - mkdir -p recipes/sglang/deepseek-v4/8k1k || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k" \ - recipes/sglang/deepseek-v4/8k1k || exit 1 -elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout main - if [[ $PRECISION == "fp4" ]]; then - mkdir -p recipes/sglang/glm5/gb300-fp4 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4" recipes/sglang/glm5/gb300-fp4 - fi -elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp8" ]]; then - # GLM-5.1 FP8 (gb300) recipes are version-controlled in-repo; overlay them - # onto the pinned submission branch. - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout sa-submission-q2-2026 - mkdir -p recipes/sglang/glm5.1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.1" recipes/sglang/glm5.1 -elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.1" ]]; then - # GLM-5.1 MTP recipe (recipes/gb300-fp4/glm5-mtp.yaml) lives on - # NVIDIA/srt-slurm:main — check it out; no in-repo overlay needed. - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout main -elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then - # Overlay our version-controlled Qwen3.5 recipes onto the srt-slurm checkout. - # fp8 recipes pin dynamo by commit hash (source install), which needs the - # cargo/maturin bootstrap included in the srt-slurm v1.0.25 release — the - # sa-submission-q2-2026 sglang install path assumes maturin ships in the - # image, and the lmsysorg/sglang nightly-dev-cu13 image doesn't include it. - # Same branch the identical gb200-fp8 recipes run on. fp4 recipes pin - # dynamo by version (pip install) and stay on the submission branch they - # were validated against. - if [[ "$USES_DCGM_POWER" == "1" ]]; then - git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout "$POWER_SRT_SLURM_PIN" || exit 1 - # The power lane must run the exact pinned producer SHA, never a moving branch. - test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } - git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" - else - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - if [[ $PRECISION == "fp8" ]]; then - git checkout v1.0.25 - else - git checkout sa-submission-q2-2026 - fi - fi - mkdir -p recipes/sglang/qwen3.5 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" recipes/sglang/qwen3.5 -elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - if [[ "${SPEC_DECODING:-}" == "mtp" ]]; then - git checkout v1.0.38 - else - git checkout sa-submission-q2-2026 - fi - mkdir -p recipes/vllm/minimax-m3-gb300-fp8 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8" recipes/vllm/minimax-m3-gb300-fp8 -elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout main - mkdir -p recipes/vllm/kimi-k2.5-fp4 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.5-fp4" recipes/vllm/kimi-k2.5-fp4 -elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "dsv4" ]]; then - # DSv4 dynamo-trt recipes use the HuggingFace model ID as model.path, - # so override SRT_SLURM_MODEL_PREFIX to match the recipe's model path key. +setup_srt_slurm "$SRT_REPO_DIR" "$FRAMEWORK" "$USES_DCGM_POWER" || exit 1 + +if [[ "$FRAMEWORK" == "dynamo-trt" && "$MODEL_PREFIX" == "dsv4" ]]; then SRT_SLURM_MODEL_PREFIX="deepseek-ai/DeepSeek-V4-Pro" - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout sa-submission-q2-2026 -elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout v1.0.72 - mkdir -p recipes/trtllm/qwen3.5/gb300-fp4/disagg - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg" \ - recipes/trtllm/qwen3.5/gb300-fp4/disagg -else - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout sa-submission-q2-2026 fi - -if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then - python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" \ - || exit 1 +# Accuracy runs use real speculative verification and a frontend colocated +# with the post-eval client on the allocation head. +if [[ "$IS_AGENTIC" == "1" && "$FRAMEWORK" == "dynamo-trt" && "$EVAL_ONLY" == "true" ]]; then + find recipes -path 'recipes/*/trtllm/*' -name '*.yaml' -exec sed -i '/TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS/d' {} + + if [[ "$MODEL_PREFIX" == "dsv4" ]]; then + SRTCTL_EVAL_ARGS+=(--set frontend.placement.node=head) + fi fi echo "Installing srtctl..." -if [[ "$IS_AGENTIC" == "1" && "$FRAMEWORK" == "dynamo-trt" && ( "$MODEL_PREFIX" == "qwen3.5" || "$MODEL_PREFIX" == "glm5.2" || "$MODEL_PREFIX" == "dsv4" ) ]]; then - sed -i 's#git clone https://github.com/ai-dynamo/dynamo.git#git clone https://github.com/cquil11/dynamo.git#' src/srtctl/core/schema.py - grep -q 'git clone https://github.com/cquil11/dynamo.git' src/srtctl/core/schema.py || exit 1 -fi export UV_INSTALL_DIR="$GITHUB_WORKSPACE/.local/bin" curl -LsSf https://astral.sh/uv/install.sh | sh export PATH="$UV_INSTALL_DIR:$PATH" @@ -591,22 +361,8 @@ if [[ "$USES_AGENTX_POWER" == "1" ]]; then "$CONFIG_PATH" "${POWER_CONCURRENCIES[@]}" || exit 1 fi -# --no-preflight skips srtctl's pre-submit model-path stat, which runs on -# the GHA runner host (im-gb300-login-02, an x86 login node). It's required -# whenever model.path resolves to the node-local /scratch NVMe that the login -# node can't see: -# - the agentic path (DSv4-Pro checkpoint), -# - glm5.1, whose GLM-5.1-NVFP4 weights are prestaged on the compute-node -# /scratch/models, and -# - qwen3.5 fp8, whose weights are also on the compute-node /scratch/models -# and which runs on srt-slurm:v1.0.25 (the release that has the preflight), -# - qwen3.5 fp4 dynamo-trt, which runs on v1.0.72 without that preflight, and -# - the qwen3.5 fp4 and dsv4 sglang power lanes, which run the pinned -# producer (a main-lineage fork that has the preflight) against the same -# /scratch checkpoints. -# The engine still fails loudly at runtime if the path is genuinely missing on -# the compute node. Other fixed-seq-len recipes resolve model.path to a -# login-visible location, so keep the precheck enforced for them. +# Skip the login-host model check for checkpoints staged only on compute +# nodes. The worker still validates the model path during startup. SRTCTL_APPLY_ARGS=( -f "$CONFIG_FILE" --tags "gb300,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" @@ -614,11 +370,8 @@ SRTCTL_APPLY_ARGS=( if [[ "$IS_AGENTIC" == "1" || "$MODEL_PREFIX" == "glm5.1" || ( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp8" ) || ( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" && ( "$FRAMEWORK" == "dynamo-trt" || "$USES_DCGM_POWER" == "1" ) ) || ( "$USES_DCGM_POWER" == "1" && "$MODEL_PREFIX" == "dsv4" && "$FRAMEWORK" == "dynamo-sglang" ) ]]; then SRTCTL_APPLY_ARGS+=(--no-preflight) fi -if [[ -n "$SRTCTL_SETUP_SCRIPT" ]]; then - SRTCTL_APPLY_ARGS+=(--setup-script "$SRTCTL_SETUP_SCRIPT") -fi -SRTCTL_OUTPUT=$(srtctl apply "${SRTCTL_APPLY_ARGS[@]}" 2>&1) +SRTCTL_OUTPUT=$(srtctl apply "${SRTCTL_EVAL_ARGS[@]}" "${SRTCTL_APPLY_ARGS[@]}" 2>&1) echo "$SRTCTL_OUTPUT" JOB_ID=$(echo "$SRTCTL_OUTPUT" | grep -oP '✅ Job \K[0-9]+' || echo "$SRTCTL_OUTPUT" | grep -oP 'Job \K[0-9]+') @@ -704,7 +457,7 @@ echo "Collecting results..." if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" "$INFMAX_WORKSPACE" \ - "$GITHUB_WORKSPACE" "$RESULT_FILENAME" "$AGENTX_POWER_SRT_SLURM_PIN" \ + "$GITHUB_WORKSPACE" "$RESULT_FILENAME" "$SRT_SLURM_COMMIT" \ "${POWER_CONCURRENCIES[@]}" || AGENTX_POWER_RC=$? fi diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index af43c740c0..4e73bfa330 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -2,7 +2,7 @@ set -e # shellcheck source=runners/slurm_utils.sh -source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" +source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" || exit 1 # System-specific configuration for H100 DGXC Slurm cluster SLURM_PARTITION="hpc-gpu-1" @@ -41,27 +41,14 @@ if [[ "$IS_MULTINODE" == "true" ]]; then exit 1 fi - echo "Cloning srt-slurm repository..." + echo "Preparing job-local srt-slurm checkout..." SRT_REPO_DIR="srt-slurm" if [ -d "$SRT_REPO_DIR" ]; then echo "Removing existing $SRT_REPO_DIR..." rm -rf "$SRT_REPO_DIR" fi - # TODO(CJQ): make first class upon srt-slurm upstream refactor - if [[ "$IS_AGENTIC" == "1" ]]; then - git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - else - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout sa-submission-q2-2026 - fi - if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then - python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" \ - || exit 1 - fi - + setup_srt_slurm "$SRT_REPO_DIR" "$FRAMEWORK" 0 || exit 1 echo "Installing srtctl..." export UV_INSTALL_DIR="/mnt/nfs/sa-shared/.uv/bin" @@ -150,7 +137,7 @@ EOF python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ "${CONFIG_FILE%%:*}" "$FRAMEWORK" || exit 1 fi - SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" --tags "h100,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) + SRTCTL_OUTPUT=$(srtctl apply "${SRTCTL_EVAL_ARGS[@]}" -f "$CONFIG_FILE" --tags "h100,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) echo "$SRTCTL_OUTPUT" # Extract JOB_ID from srtctl output diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index e7bfbeff27..3284aadb8b 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -7,17 +7,10 @@ SLURM_ACCOUNT="sa-shared" HF_HUB_CACHE_MOUNT="${HF_HUB_CACHE_MOUNT:-/models/gharunners/hf-hub-cache}" AIPERF_MMAP_CACHE_HOST_PATH="${AIPERF_MMAP_CACHE_HOST_PATH:-/home/sa-shared/gharunners/ai-perf-cache}" -# Immutable producer prerequisite for the GLM-5.2 AgentX lane. This fork is -# intentionally long-lived; update the SHA only after reviewing a new fork -# commit and re-running the H200 hardware gate. -POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" -POWER_SRT_SLURM_PIN="e5c837f06a362dc888dfea2ee588e9f19c298270" -AGENTX_POWER_SRT_SLURM_PIN="80d7203e424f903c9017de4608ee2044afce9574" -SELECTED_POWER_SRT_SLURM_PIN="$POWER_SRT_SLURM_PIN" set -x -source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" +source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" || exit 1 if [[ "$IS_MULTINODE" == "true" ]]; then @@ -28,15 +21,14 @@ if [[ "$IS_MULTINODE" == "true" ]]; then CONFIG_PATH="${CONFIG_FILE%%:*}" LOCAL_CONFIG_FILE="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${CONFIG_PATH#recipes/}" - # The producer pin decision is recipe-driven. Upstream-only recipes have - # no workspace mirror and remain non-power. + # Power collection is enabled by the selected recipe. USES_DCGM_POWER=0 _RECIPE_REL="${CONFIG_FILE%%:*}" _RECIPE_SRC="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${_RECIPE_REL#recipes/}" if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' /^telemetry:/ { t = 1; next } t && /^[^ ]/ { t = 0 } - t && /^ provider: dcgm-power$/ { p = 1 } + t && /^ dcgm_exporter:/ { p = 1 } t && /^ enabled: true$/ { e = 1 } END { exit !(p && e) } ' "$_RECIPE_SRC"; then @@ -47,7 +39,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "kimik3" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "vllm" ]]; then USES_KIMIK3_POWER=1 - SELECTED_POWER_SRT_SLURM_PIN="$AGENTX_POWER_SRT_SLURM_PIN" + elif [[ "$USES_DCGM_POWER" == "1" && ( "$IS_AGENTIC" != "1" || "$FRAMEWORK" != "dynamo-sglang" || @@ -105,66 +97,14 @@ if [[ "$IS_MULTINODE" == "true" ]]; then exit 1 fi - echo "Cloning srt-slurm repository..." + echo "Preparing job-local srt-slurm checkout..." SRT_REPO_DIR="srt-slurm" if [ -d "$SRT_REPO_DIR" ]; then echo "Removing existing $SRT_REPO_DIR..." rm -rf "$SRT_REPO_DIR" fi - if [[ $IS_AGENTIC == "1" && $FRAMEWORK == "dynamo-sglang" && ( - "$MODEL_PREFIX" == "glm5.2" || "$MODEL_PREFIX" == "dsv4" - ) ]]; then - if [[ "$USES_DCGM_POWER" == "1" ]]; then - # The pinned fork carries the v1.0.44 AgentX lifecycle plus the formal - # custom-benchmark dcgm-power contract used by the allowlisted recipes. - git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout "$POWER_SRT_SLURM_PIN" || exit 1 - test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } - git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" - elif [[ "$MODEL_PREFIX" == "dsv4" ]]; then - # Non-power DSV4 runs keep the upstream release their perf-changelog - # provenance records. v1.0.38 also injects every logical SGLang worker - # leader's /metrics URL into AIPERF_SERVER_METRICS_URLS for custom - # benchmarks; v1.0.10 wired that only for built-in AIPerf runners, so - # the AgentX trace artifacts came back with no backend engine series - # behind them. - git clone --branch v1.0.38 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - else - # v1.0.44 includes the AgentX custom benchmark integration and passes - # every logical SGLang worker's Prometheus URL to AIPerf. - git clone --branch v1.0.44 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - fi - elif [[ $IS_AGENTIC == "1" && $FRAMEWORK == "vllm" && $MODEL_PREFIX == "kimik3" ]]; then - if [[ "$USES_KIMIK3_POWER" == "1" ]]; then - git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout "$SELECTED_POWER_SRT_SLURM_PIN" - test "$(git rev-parse HEAD)" = "$SELECTED_POWER_SRT_SLURM_PIN" || exit 1 - git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" - else - git clone https://github.com/functionstackx/srt-slurm-nv.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout df5baa93f4caf5169dea2a4236ad2cc742fe40e7 - fi - mkdir -p recipes/vllm/kimi-k3/agentic - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ - recipes/vllm/kimi-k3/agentic - elif [[ "$IS_AGENTIC" == "1" ]]; then - git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - else - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout sa-submission-q2-2026 - fi - if [[ "${EVAL_FRAMEWORK:-lm-eval}" != "lm-eval" ]]; then - python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" \ - || exit 1 - fi + setup_srt_slurm "$SRT_REPO_DIR" "$FRAMEWORK" "$USES_DCGM_POWER" || exit 1 echo "Installing srtctl..." curl -LsSf https://astral.sh/uv/install.sh | sh @@ -339,7 +279,7 @@ EOF if [[ "$IS_AGENTIC" == "1" ]]; then WORKLOAD_TAG="agentic" fi - SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" --tags "h200,${MODEL_PREFIX},${PRECISION},${WORKLOAD_TAG},infmax-$(date +%Y%m%d)" 2>&1) + SRTCTL_OUTPUT=$(srtctl apply "${SRTCTL_EVAL_ARGS[@]}" -f "$CONFIG_FILE" --tags "h200,${MODEL_PREFIX},${PRECISION},${WORKLOAD_TAG},infmax-$(date +%Y%m%d)" 2>&1) echo "$SRTCTL_OUTPUT" # Extract JOB_ID from srtctl output @@ -383,7 +323,7 @@ EOF read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" \ "$GITHUB_WORKSPACE" "$GITHUB_WORKSPACE" "$RESULT_FILENAME" \ - "$SELECTED_POWER_SRT_SLURM_PIN" "${POWER_CONCURRENCIES[@]}" || AGENTX_POWER_RC=$? + "$SRT_SLURM_COMMIT" "${POWER_CONCURRENCIES[@]}" || AGENTX_POWER_RC=$? elif [[ "$USES_DCGM_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then POWER_LOGS_ROOT=$(cd "$LOGS_DIR" && pwd -P) read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" @@ -393,7 +333,7 @@ EOF --agg-result "$GITHUB_WORKSPACE/${RESULT_FILENAME}_conc${concurrency}.json" --power-dir "$POWER_LOGS_ROOT/power" --logs-root "$POWER_LOGS_ROOT" - --expected-producer-sha "$SELECTED_POWER_SRT_SLURM_PIN" + --expected-producer-sha "$SRT_SLURM_COMMIT" ) case "${REQUIRE_POWER:-0}" in 1|true|TRUE|yes|YES) power_args+=(--require-power) ;; diff --git a/runners/patch_srt_eval_dispatch.py b/runners/patch_srt_eval_dispatch.py deleted file mode 100755 index 206e9c0133..0000000000 --- a/runners/patch_srt_eval_dispatch.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python3 -"""Enable InferenceX-selected eval dispatch in an srt-slurm checkout.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -DO_SWEEP_ENV_BLOCK = """ "EVAL_ONLY", - "IS_MULTINODE",""" -DO_SWEEP_ENV_REPLACEMENT = """ "EVAL_ONLY", - "EVAL_FRAMEWORK", - "EVAL_CONC", - "EVAL_LIMIT", - "EVAL_SUITE", - "SWEBENCH_GEN_MODE", - "SWEBENCH_USE_MODAL", - "MODAL_TOKEN_ID", - "MODAL_TOKEN_SECRET", - "IS_AGENTIC", - "SCENARIO_TYPE", - "IS_MULTINODE",""" -LM_EVAL_COMMAND = 'run_eval --framework lm-eval --port "$PORT" || eval_rc=$?' -GENERIC_EVAL_COMMAND = 'run_eval --port "$PORT" || eval_rc=$?' -EVAL_ARTIFACT_COPY = """cp -v results*.json /logs/eval_results/ 2>/dev/null || true -cp -v sample*.jsonl /logs/eval_results/ 2>/dev/null || true""" -VERIFIER_ARTIFACT_COPY = 'stage_eval_artifacts /logs/eval_results "$PWD" || true' - - -def prepare_replacements( - path: Path, - replacements: tuple[tuple[str, str], ...], -) -> tuple[str, str, bool]: - """Validate source replacements without mutating the checkout.""" - original = path.read_text() - content = original - changed = False - for old, new in replacements: - old_count = content.count(old) - new_count = content.count(new) - if old_count == 1 and new_count == 0: - anchor = old.splitlines()[0] - anchor_count = content.count(anchor) - if anchor_count != 1: - raise RuntimeError( - f"invalid patch state in {path}: " - f"anchor {anchor!r} count={anchor_count}" - ) - content = content.replace(old, new, 1) - changed = True - elif old_count != 0 or new_count != 1: - raise RuntimeError( - f"invalid patch state in {path}: old anchor count={old_count}, " - f"replacement count={new_count}" - ) - return original, content, changed - - -def patch_checkout(root: Path) -> list[Path]: - """Patch both post-eval sources after validating the complete checkout.""" - patches = ( - ( - root / "src/srtctl/cli/do_sweep.py", - ((DO_SWEEP_ENV_BLOCK, DO_SWEEP_ENV_REPLACEMENT),), - ), - ( - root / "src/srtctl/benchmarks/scripts/lm-eval/bench.sh", - ( - (LM_EVAL_COMMAND, GENERIC_EVAL_COMMAND), - (EVAL_ARTIFACT_COPY, VERIFIER_ARTIFACT_COPY), - ), - ), - ) - staged = [ - (path, *prepare_replacements(path, replacements)) - for path, replacements in patches - ] - changed = [] - written = [] - try: - for path, original, replacement, needs_write in staged: - if needs_write: - path.write_text(replacement) - written.append((path, original)) - changed.append(path) - except OSError: - for path, original in reversed(written): - path.write_text(original) - raise - return changed - - -def main(argv: list[str]) -> int: - """Patch the checkout named on the command line.""" - if len(argv) != 2: - print(f"Usage: {argv[0]} SRT_SLURM_CHECKOUT", file=sys.stderr) - return 2 - root = Path(argv[1]).resolve() - try: - changed = patch_checkout(root) - except (OSError, RuntimeError) as error: - print( - f"ERROR: failed to patch srt-slurm eval dispatch: {error}", file=sys.stderr - ) - return 1 - if changed: - for path in changed: - print(f"Patched srt-slurm eval dispatch: {path}") - else: - print("srt-slurm eval dispatch is already patched") - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/runners/patch_srt_vllm_dp_ranks.py b/runners/patch_srt_vllm_dp_ranks.py deleted file mode 100755 index 9decf0f06e..0000000000 --- a/runners/patch_srt_vllm_dp_ranks.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python3 -"""Patch srt-slurm v1.0.53 to allocate one process per vLLM DP rank.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -OLD_BLOCK = ''' else: - # DP+EP mode: one process per GPU - # Each process gets a single GPU and a unique dp_rank - dp_rank = 0 - # Allocate a unique DP RPC port for this endpoint's leader node - dp_rpc_port = port_allocator.next_dp_rpc_port(endpoint.leader_node) - # Allocate a single NIXL base port for this endpoint. - # vLLM internally computes: actual_port = base + data_parallel_rank - # so all DP ranks in the endpoint share the same base port. - dp_size = self._get_dp_size(endpoint.mode) or len(endpoint.gpu_indices) - nixl_base_port = port_allocator.next_nixl_port_block(dp_size) - for _node_rank, node in enumerate(endpoint.nodes): - for gpu_idx in sorted(endpoint.gpu_indices): - is_leader = dp_rank == 0 - http_port = port_allocator.next_http_port(node) if is_leader else 0 - bootstrap_port = ( - port_allocator.next_bootstrap_port(node) - if endpoint.mode == "prefill" and is_leader - else None - ) - kv_events_port = port_allocator.next_kv_events_port() - nixl_port = nixl_base_port - - processes.append( - Process( - node=node, - gpu_indices=frozenset([gpu_idx]), # Single GPU per process - sys_port=current_sys_port, - http_port=http_port, - endpoint_mode=endpoint.mode, - endpoint_index=endpoint.index, - node_rank=dp_rank, # dp_rank stored in node_rank for now - bootstrap_port=bootstrap_port, - kv_events_port=kv_events_port, - nixl_port=nixl_port, - dp_rpc_port=dp_rpc_port, - ) - ) - current_sys_port += 1 - dp_rank += 1 -''' - -NEW_BLOCK = ''' else: - # External DP mode: one process per DP rank. A rank may own - # multiple GPUs when tensor or pipeline parallelism is enabled. - dp_rank = 0 - dp_rpc_port = port_allocator.next_dp_rpc_port(endpoint.leader_node) - config = self.get_config_for_mode(endpoint.mode) - dp_size = self._get_dp_size(endpoint.mode) or endpoint.total_gpus - tp_size = config.get("tensor-parallel-size") or config.get("tensor_parallel_size") or 1 - pp_size = config.get("pipeline-parallel-size") or config.get("pipeline_parallel_size") or 1 - gpus_per_dp_rank = tp_size * pp_size - expected_gpus = dp_size * gpus_per_dp_rank - if endpoint.total_gpus != expected_gpus: - raise ValueError( - f"{endpoint.mode} DP={dp_size}, TP={tp_size}, PP={pp_size} requires " - f"{expected_gpus} GPUs, but the endpoint allocated {endpoint.total_gpus}" - ) - - nixl_base_port = port_allocator.next_nixl_port_block(dp_size) - for node in endpoint.nodes: - local_gpus = sorted(endpoint.gpu_indices) - if len(local_gpus) % gpus_per_dp_rank != 0: - raise ValueError( - f"{endpoint.mode} TP={tp_size}, PP={pp_size} requires " - f"{gpus_per_dp_rank} GPUs per DP rank, but node {node} has " - f"{len(local_gpus)} allocated GPUs" - ) - for offset in range(0, len(local_gpus), gpus_per_dp_rank): - rank_gpus = frozenset(local_gpus[offset : offset + gpus_per_dp_rank]) - is_leader = dp_rank == 0 - http_port = port_allocator.next_http_port(node) if is_leader else 0 - bootstrap_port = ( - port_allocator.next_bootstrap_port(node) - if endpoint.mode == "prefill" and is_leader - else None - ) - - processes.append( - Process( - node=node, - gpu_indices=rank_gpus, - sys_port=current_sys_port, - http_port=http_port, - endpoint_mode=endpoint.mode, - endpoint_index=endpoint.index, - node_rank=dp_rank, - bootstrap_port=bootstrap_port, - kv_events_port=port_allocator.next_kv_events_port(), - nixl_port=nixl_base_port, - dp_rpc_port=dp_rpc_port, - ) - ) - current_sys_port += 1 - dp_rank += 1 - - if dp_rank != dp_size: - raise ValueError( - f"{endpoint.mode} allocated {dp_rank} DP ranks, expected {dp_size}" - ) -''' - - -def patch_backend(root: Path) -> bool: - """Apply the rank allocator patch and return whether the file changed.""" - backend = root / "src/srtctl/backends/vllm.py" - source = backend.read_text() - - if NEW_BLOCK in source: - return False - if source.count(OLD_BLOCK) != 1: - raise RuntimeError( - f"unsupported srt-slurm vLLM backend at {backend}: expected allocation block not found exactly once" - ) - - backend.write_text(source.replace(OLD_BLOCK, NEW_BLOCK)) - return True - - -def main(argv: list[str]) -> int: - if len(argv) != 2: - print(f"Usage: {argv[0]} SRT_SLURM_CHECKOUT", file=sys.stderr) - return 2 - - try: - changed = patch_backend(Path(argv[1]).resolve()) - except (OSError, RuntimeError) as error: - print(f"ERROR: failed to patch srt-slurm vLLM DP ranks: {error}", file=sys.stderr) - return 1 - - state = "Patched" if changed else "Already patched" - print(f"{state} srt-slurm vLLM DP rank allocation") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv)) diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 49f33f6b22..d35d977090 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -3,6 +3,49 @@ # Launchers source this file before changing into srt-slurm. INFERENCEX_SLURM_UTILS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SRTCTL_EVAL_ARGS=( + --set 'post_eval.command=["bash", "{infmax_workspace}/benchmarks/multi_node/srt_eval.sh", "{endpoint}", "{infmax_workspace}"]' + --set 'post_eval.passthrough_env=["EVAL_FRAMEWORK", "EVAL_CONC", "EVAL_LIMIT", "EVAL_SUITE", "SWEBENCH_GEN_MODE", "SWEBENCH_USE_MODAL", "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET", "IS_AGENTIC", "SCENARIO_TYPE"]' +) + +# Leaves the caller in the checkout, matching the launchers' installation flow. +# Every recipe is owned by InferenceX; srt-slurm 2 no longer ships recipes/. +setup_srt_slurm() { + if [[ $# -ne 3 || -z "$1" || -z "$2" || ( "$3" != 0 && "$3" != 1 ) ]]; then + echo "Usage: setup_srt_slurm destination framework uses_power (0 or 1)" >&2 + return 1 + fi + local destination="$1" framework="$2" uses_power="$3" + local source="$INFERENCEX_SLURM_UTILS_DIR/../utils/srt-slurm" + if [[ "$framework" == "tilert" ]]; then + # Sole fork exception until NVIDIA supports the TileRT backend and router. + SRT_SLURM_COMMIT=6bc3f306bdafa1edfb5dded2fcda8f1ccede1bde + git init "$destination" || return 1 + git -C "$destination" remote add origin https://github.com/SemiAnalysisAI/srt-slurm.git || return 1 + git -C "$destination" fetch --depth=1 origin "$SRT_SLURM_COMMIT" || return 1 + git -C "$destination" checkout --detach "$SRT_SLURM_COMMIT" || return 1 + else + if [[ ! -e "$source/.git" ]]; then + echo "Missing srt-slurm submodule; run git submodule update --init before launching." >&2 + return 1 + fi + SRT_SLURM_COMMIT=$(git -C "$source" rev-parse HEAD) || return 1 + # A local clone keeps job writes isolated and preserves upstream Git provenance. + git clone --no-hardlinks "$source" "$destination" || return 1 + fi + cd "$destination" || return 1 + [[ "$(git rev-parse HEAD)" == "$SRT_SLURM_COMMIT" ]] || return 1 + git rev-parse HEAD > "$GITHUB_WORKSPACE/srt-slurm-sha.txt" || return 1 + if [[ "$uses_power" == "1" ]]; then + cp "$GITHUB_WORKSPACE/srt-slurm-sha.txt" "$GITHUB_WORKSPACE/power-producer-sha.txt" || return 1 + fi + mkdir -p recipes benchmarks/multi_node || return 1 + cp -R "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/." recipes/ || return 1 + # Both CONFIG_FILE spellings currently occur in master configs. + ln -s ../../recipes benchmarks/multi_node/srt-slurm-recipes || return 1 + cp -R "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/." configs/ || return 1 +} + # Use the requested image's cache identity, never a convenient older squash file. resolve_h100_srt_container() { local image="$1" framework="$2" diff --git a/runners/synthetic_injectors/_roles.py b/runners/synthetic_injectors/_roles.py new file mode 100644 index 0000000000..ee833a6a36 --- /dev/null +++ b/runners/synthetic_injectors/_roles.py @@ -0,0 +1,41 @@ +"""Preserve recipe formatting while injecting environment into schema-2 roles.""" + +import re + + +def rewrite_role_environments(content: str, values: tuple[tuple[str, str], ...]) -> tuple[str, int]: + """Set worker variables without touching frontend, service or client env.""" + # Restrict schema-2 injection to worker roles, excluding frontend, + # benchmark and service env blocks. Keep comments and YAML anchors. + roles = re.search(r"(?ms)^roles:\n.*?(?=^\S|\Z)", content) + if roles is None: + return content, 0 + count = 0 + + def inject_role(match: re.Match[str]) -> str: + nonlocal count + block = match.group(0) + env = re.search(r"(?m)^ env:([^\n]*)$", block) + variables = "".join(f'\n {key}: "{value}"' for key, value in values) + if env: + suffix = env.group(1).strip() + if suffix.startswith("*"): + replacement = " env:\n <<: " + suffix + variables + elif not suffix or suffix.startswith(("&", "#")): + replacement = env.group(0) + variables + else: + raise ValueError("synthetic acceptance requires a block-style role env") + block = block[:env.start()] + replacement + block[env.end():] + else: + header_end = block.index("\n") + block = block[:header_end] + "\n env:" + variables + block[header_end:] + count += 1 + return block + + rewritten_roles = re.sub( + r"(?ms)^ (?:agg|prefill|decode):\n.*?(?=^ \S|\Z)", + inject_role, + roles.group(0), + ) + rewritten = content[:roles.start()] + rewritten_roles + content[roles.end():] + return rewritten, count diff --git a/runners/synthetic_injectors/sglang.py b/runners/synthetic_injectors/sglang.py index d5cd6e0b16..c2a8758a5b 100644 --- a/runners/synthetic_injectors/sglang.py +++ b/runners/synthetic_injectors/sglang.py @@ -4,9 +4,9 @@ import sys from . import register +from ._roles import rewrite_role_environments _SPEC_STEPS_RE = re.compile(r"(?m)^\s+speculative-num-steps:\s*([0-9]+)\s*$") -_ENV_BLOCK_RE = re.compile(r"(?m)^( (?:aggregated|prefill|decode)_environment:\s*)$") _SIMULATED_ACCEPTANCE_ENV_RE = re.compile( r"(?m)^[ \t]+SGLANG_SIMULATE_ACC_(?:LEN|METHOD|TOKEN_MODE):[^\n]*(?:\n|$)" ) @@ -23,15 +23,12 @@ def rewrite(content, al, log): if "SGLANG_SIMULATE_ACC_LEN" in content: raise ValueError("recipe already contains SGLANG_SIMULATE_ACC_* variables") - variables = ( - f'\n SGLANG_SIMULATE_ACC_LEN: "{al:g}"' - '\n SGLANG_SIMULATE_ACC_METHOD: "match-expected"' - '\n SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token"' - ) - rewritten, count = _ENV_BLOCK_RE.subn( - lambda match: match.group(1) + variables, - content, + values = ( + ("SGLANG_SIMULATE_ACC_LEN", f"{al:g}"), + ("SGLANG_SIMULATE_ACC_METHOD", "match-expected"), + ("SGLANG_SIMULATE_ACC_TOKEN_MODE", "real-draft-token"), ) + rewritten, count = rewrite_role_environments(content, values) if count: log(f"Added SGLANG_SIMULATE_ACC_* to {count} worker environment block(s)") return rewritten, count diff --git a/runners/synthetic_injectors/trtllm.py b/runners/synthetic_injectors/trtllm.py index 021c2d613b..c67cb46b92 100644 --- a/runners/synthetic_injectors/trtllm.py +++ b/runners/synthetic_injectors/trtllm.py @@ -4,7 +4,7 @@ ``TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS``. Its value is the number of *draft* tokens accepted per step, so an acceptance length AL (target token + accepted drafts) maps to ``AL - 1``. Throughput opt-ins inject that variable -into every ``*_environment`` block under ``backend:``; eval-only runs remove it +into each worker's ``roles..env``; eval-only runs remove it so the verifier's real acceptance drives the generated text. The backend is registered for both direct trtllm-serve and Dynamo-TRT-LLM recipes. """ @@ -13,10 +13,9 @@ import sys from . import register +from ._roles import rewrite_role_environments _ENV_KEY = "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS" -# A `*_environment:` mapping header under backend (aggregated/prefill/decode). -_ENV_BLOCK_RE = re.compile(r"^(?P[ \t]*)(?:aggregated|prefill|decode)_environment:[ \t]*$") _ENV_LINE_RE = re.compile(rf"^[ \t]*{_ENV_KEY}:.*\n?", re.MULTILINE) _DRAFT_LEN_RE = re.compile(r"^\s*(?:max_draft_len|num_nextn_predict_layers):\s*(\d+)", re.MULTILINE) @@ -32,7 +31,7 @@ def _format_value(al): def rewrite(content, al, log): - """Force ``AL - 1`` accepted draft tokens in every backend environment block. + """Force ``AL - 1`` accepted draft tokens in every worker role environment. Returns ``(new_content, count)`` where count is the number of environment blocks now carrying the variable (0 => no block found, recipe left unchanged). @@ -42,29 +41,10 @@ def rewrite(content, al, log): # variable ends up with exactly one line per block. content = _ENV_LINE_RE.sub("", content) - lines = content.splitlines(keepends=True) - out, count, i = [], 0, 0 - while i < len(lines): - line = lines[i] - out.append(line) - m = _ENV_BLOCK_RE.match(line.rstrip("\n")) - if m: - # Child indentation comes from the first non-empty following line; - # fall back to two spaces past the header. - child_indent = None - for nxt in lines[i + 1:]: - if nxt.strip(): - child_indent = nxt[: len(nxt) - len(nxt.lstrip())] - break - if child_indent is None or len(child_indent) <= len(m.group("indent")): - child_indent = m.group("indent") + " " - out.append(f"{child_indent}{_ENV_KEY}: '{value}'\n") - count += 1 - i += 1 - new_content = "".join(out) + rewritten, count = rewrite_role_environments(content, ((_ENV_KEY, value),)) if count: - log(f"Set {_ENV_KEY}={value} (AL={al}) in {count} environment block(s)") - return new_content, count + log(f"Set {_ENV_KEY}={value} (AL={al}) in {count} role environment block(s)") + return rewritten, count def rewrite_real(content, log): diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index d92076ebe3..9a12f274ac 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -10,8 +10,6 @@ REPO_ROOT = Path(__file__).resolve().parents[1] SLURM_UTILS = REPO_ROOT / "runners" / "slurm_utils.sh" -PATCH_SRT_EVAL = REPO_ROOT / "runners" / "patch_srt_eval_dispatch.py" -PATCH_SRT_DP_RANKS = REPO_ROOT / "runners" / "patch_srt_vllm_dp_ranks.py" PATCH_TRTLLM_CHAT_STORE = REPO_ROOT / "runners" / "patch_trtllm_chat_store.py" PATCH_VLLM_SIMPLE_KV = REPO_ROOT / "runners" / "patch_vllm_simple_kv_offload.py" INJECT_ACCEPTANCE = REPO_ROOT / "runners" / "inject_synthetic_acceptance.py" @@ -212,134 +210,6 @@ def test_copy_agentic_results_fails_when_aggregate_is_missing( assert "no run_conc*.json results found" in result.stderr -def test_b300_dsxe_draft_model_uses_public_paths_and_writable_hf_cache() -> None: - launcher = (REPO_ROOT / "runners/launch_b300-dsxe.sh").read_text() - - assert 'SLURM_ACCOUNT="benchmark"' in launcher - assert 'SQUASH_DIR="/data/home/sa-gha-runner/squash"' in launcher - assert 'SHARED_MODEL_ROOT="/data/models"' in launcher - assert 'WRITABLE_MODELS_DIR="/data/home/sa-gha-runner/models"' in launcher - assert '[[ "$MODEL_BASENAME" == "DeepSeek-V4-Pro-0813" ]]' in launcher - assert 'MODEL_MOUNT_DIR="$SHARED_MODEL_ROOT"' in launcher - assert "nv-gha-runner" not in launcher - assert ( - '[[ "$SPEC_DECODING" == "mtp" || "$SPEC_DECODING" == "draft_model" ]]' - in launcher - ) - assert 'export HF_HOME="$HF_CACHE_CONTAINER_DIR"' in launcher - assert 'export HF_HUB_CACHE="$HF_CACHE_CONTAINER_DIR/hub"' in launcher - assert 'export HF_XET_CACHE="$HF_CACHE_CONTAINER_DIR/xet"' in launcher - assert '"$HF_CACHE_HOST_DIR:$HF_CACHE_CONTAINER_DIR"' in launcher - - -def test_patch_srt_eval_dispatch_forwards_selection_and_is_idempotent( - tmp_path: Path, -) -> None: - do_sweep = tmp_path / "src/srtctl/cli/do_sweep.py" - eval_script = tmp_path / "src/srtctl/benchmarks/scripts/lm-eval/bench.sh" - do_sweep.parent.mkdir(parents=True) - eval_script.parent.mkdir(parents=True) - do_sweep.write_text( - "def forwarded(environment):\n" - " forwarded = {}\n" - " if environment:\n" - " for var in [\n" - ' "RUN_EVAL",\n' - ' "EVAL_ONLY",\n' - ' "IS_MULTINODE",\n' - " ]:\n" - " if var in environment:\n" - " forwarded[var] = environment[var]\n" - " return forwarded\n" - ) - eval_script.write_text( - 'run_eval --framework lm-eval --port "$PORT" || eval_rc=$?\n' - "cp -v results*.json /logs/eval_results/ 2>/dev/null || true\n" - "cp -v sample*.jsonl /logs/eval_results/ 2>/dev/null || true\n" - ) - - first = subprocess.run( - ["python3", str(PATCH_SRT_EVAL), str(tmp_path)], - check=False, - capture_output=True, - text=True, - ) - patched_sweep = do_sweep.read_text() - patched_eval = eval_script.read_text() - second = subprocess.run( - ["python3", str(PATCH_SRT_EVAL), str(tmp_path)], - check=False, - capture_output=True, - text=True, - ) - - assert first.returncode == 0, first.stderr - assert second.returncode == 0, second.stderr - assert do_sweep.read_text() == patched_sweep - assert eval_script.read_text() == patched_eval - settings = {name: f"value-{name}" for name in ( - "EVAL_FRAMEWORK", "EVAL_SUITE", "EVAL_CONC", "EVAL_LIMIT", - "SWEBENCH_GEN_MODE", "SWEBENCH_USE_MODAL", "MODAL_TOKEN_ID", - "MODAL_TOKEN_SECRET", "IS_AGENTIC", "SCENARIO_TYPE", - )} - forwarded = runpy.run_path(str(do_sweep))["forwarded"] - assert forwarded({**settings, "UNRELATED": "do not forward"}) == settings - - execution = run_bash( - 'PORT=12345; run_eval() { printf "eval:%s\\n" "$*"; }; ' - 'stage_eval_artifacts() { printf "stage:%s\\n" "$1"; }; source "$1"', - eval_script, - ) - assert execution.returncode == 0, execution.stderr - assert execution.stdout.splitlines() == ["eval:--port 12345", "stage:/logs/eval_results"] - - -def test_patch_srt_vllm_dp_ranks_is_idempotent_and_preserves_surrounding_code( - tmp_path: Path, -) -> None: - symbols = runpy.run_path(str(PATCH_SRT_DP_RANKS)) - backend = tmp_path / "src/srtctl/backends/vllm.py" - backend.parent.mkdir(parents=True) - original = f"prefix\n{symbols['OLD_BLOCK']}suffix\n" - backend.write_text(original) - - first = subprocess.run( - ["python3", str(PATCH_SRT_DP_RANKS), str(tmp_path)], - check=False, - capture_output=True, - text=True, - ) - patched = backend.read_text() - second = subprocess.run( - ["python3", str(PATCH_SRT_DP_RANKS), str(tmp_path)], - check=False, - capture_output=True, - text=True, - ) - - assert first.returncode == 0, first.stderr - assert second.returncode == 0, second.stderr - assert patched != original - assert patched.startswith("prefix\n") and patched.endswith("suffix\n") - assert backend.read_text() == patched - - -def test_patch_srt_vllm_dp_ranks_rejects_unknown_source(tmp_path: Path) -> None: - backend = tmp_path / "src/srtctl/backends/vllm.py" - backend.parent.mkdir(parents=True) - backend.write_text("unsupported backend\n") - - result = subprocess.run( - ["python3", str(PATCH_SRT_DP_RANKS), str(tmp_path)], - check=False, - capture_output=True, - text=True, - ) - - assert result.returncode == 1 - assert backend.read_text() == "unsupported backend\n" - - def test_patch_trtllm_chat_store_accepts_false_and_is_idempotent( tmp_path: Path, ) -> None: @@ -439,62 +309,6 @@ def test_patch_vllm_simple_kv_offload_rejects_unknown_source( assert worker.read_text() == "unsupported worker\n" -def test_patch_srt_eval_dispatch_preflights_before_writing(tmp_path: Path) -> None: - do_sweep = tmp_path / "src/srtctl/cli/do_sweep.py" - eval_script = tmp_path / "src/srtctl/benchmarks/scripts/lm-eval/bench.sh" - do_sweep.parent.mkdir(parents=True) - eval_script.parent.mkdir(parents=True) - original_do_sweep = ' "EVAL_ONLY",\n "IS_MULTINODE",\n' - original_eval_script = "unsupported eval hook\n" - do_sweep.write_text(original_do_sweep) - eval_script.write_text(original_eval_script) - - result = subprocess.run( - ["python3", str(PATCH_SRT_EVAL), str(tmp_path)], - check=False, - capture_output=True, - text=True, - ) - - assert result.returncode == 1 - assert do_sweep.read_text() == original_do_sweep - assert eval_script.read_text() == original_eval_script - - -def test_patch_srt_eval_dispatch_rejects_mixed_patch_state(tmp_path: Path) -> None: - do_sweep = tmp_path / "src/srtctl/cli/do_sweep.py" - eval_script = tmp_path / "src/srtctl/benchmarks/scripts/lm-eval/bench.sh" - do_sweep.parent.mkdir(parents=True) - eval_script.parent.mkdir(parents=True) - original_do_sweep = ( - ' "EVAL_ONLY",\n' - ' "IS_MULTINODE",\n' - ' "EVAL_ONLY",\n' - ' "EVAL_FRAMEWORK",\n' - ' "EVAL_SUITE",\n' - ' "IS_MULTINODE",\n' - ) - original_eval_script = ( - 'run_eval --framework lm-eval --port "$PORT" || eval_rc=$?\n' - "cp -v results*.json /logs/eval_results/ 2>/dev/null || true\n" - "cp -v sample*.jsonl /logs/eval_results/ 2>/dev/null || true\n" - ) - do_sweep.write_text(original_do_sweep) - eval_script.write_text(original_eval_script) - - result = subprocess.run( - ["python3", str(PATCH_SRT_EVAL), str(tmp_path)], - check=False, - capture_output=True, - text=True, - ) - - assert result.returncode == 1 - assert "invalid patch state" in result.stderr - assert do_sweep.read_text() == original_do_sweep - assert eval_script.read_text() == original_eval_script - - def test_eval_only_restores_real_vllm_acceptance(tmp_path: Path) -> None: recipe = tmp_path / "recipe.yaml" recipe.write_text( @@ -526,9 +340,9 @@ def test_eval_only_restores_real_vllm_acceptance(tmp_path: Path) -> None: def test_eval_only_removes_sglang_simulated_acceptance(tmp_path: Path) -> None: recipe = tmp_path / "recipe.yaml" recipe.write_text( - "backend:\n" - " sglang_config:\n" - " decode_environment:\n" + "schema: 2\nengine: sglang\nroles:\n" + " decode:\n" + " env:\n" ' SGLANG_SIMULATE_ACC_LEN: "2.99"\n' ' SGLANG_SIMULATE_ACC_METHOD: "match-expected"\n' ' SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token"\n' @@ -544,7 +358,7 @@ def test_eval_only_removes_sglang_simulated_acceptance(tmp_path: Path) -> None: ) assert result.returncode == 0, result.stderr - environment = yaml.safe_load(recipe.read_text())["backend"]["sglang_config"]["decode_environment"] + environment = yaml.safe_load(recipe.read_text())["roles"]["decode"]["env"] assert environment == {"KEEP_ME": "unchanged"} @@ -553,12 +367,13 @@ def test_sglang_throughput_rejects_existing_simulated_acceptance( ) -> None: recipe = tmp_path / "recipe.yaml" original = ( - "backend:\n" - " aggregated_environment:\n" - ' SGLANG_SIMULATE_ACC_LEN: "2.99"\n' - ' SGLANG_SIMULATE_ACC_METHOD: "match-expected"\n' - ' SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token"\n' - " KEEP_ME: unchanged\n" + "schema: 2\nengine: sglang\nroles:\n" + " agg:\n" + " env:\n" + ' SGLANG_SIMULATE_ACC_LEN: "2.99"\n' + ' SGLANG_SIMULATE_ACC_METHOD: "match-expected"\n' + ' SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token"\n' + " KEEP_ME: unchanged\n" ) recipe.write_text(original) @@ -583,7 +398,7 @@ def test_eval_only_acceptance_rewrite_allows_non_speculative_recipe( tmp_path: Path, ) -> None: recipe = tmp_path / "recipe.yaml" - original = "backend:\n type: vllm\n" + original = "schema: 2\nengine: vllm\nroles:\n agg:\n nodes: 1\n" recipe.write_text(original) result = subprocess.run( @@ -650,4 +465,3 @@ def test_mi355x_agentic_model_mount_and_routing( ) in args script = f"benchmarks/single_node/agentic/{prefix}_fp4_mi355x_vllm_mtp.sh" assert args[-2] == script - assert (REPO_ROOT / script).is_file() diff --git a/runners/test_synthetic_injectors.py b/runners/test_synthetic_injectors.py index cc72dbf7fe..3da28dc4a1 100644 --- a/runners/test_synthetic_injectors.py +++ b/runners/test_synthetic_injectors.py @@ -3,18 +3,22 @@ import os import sys +import yaml + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from synthetic_injectors import get_injector # noqa: E402 -TRTLLM_RECIPE = """name: dynamo-agg-gb200-tp4-c20-b1-eagle3 -backend: - type: trtllm - aggregated_environment: - HF_HUB_OFFLINE: '1' - TRTLLM_ENABLE_PDL: '1' - trtllm_config: - aggregated: +TRTLLM_RECIPE = """schema: 2 +name: dynamo-agg-gb200-tp4-c20-b1-eagle3 +engine: trtllm +roles: + agg: + nodes: 1 + env: + HF_HUB_OFFLINE: '1' + TRTLLM_ENABLE_PDL: '1' + args: speculative_config: decoding_type: Eagle3 max_draft_len: 3 @@ -37,23 +41,22 @@ def test_trtllm_rewrite_injects_al_minus_one_into_environment(): injector = get_injector("dynamo-trt") new, count = injector.rewrite(TRTLLM_RECIPE, 2.78, _noop) assert count == 1 - assert " TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.78'\n" in new - # Inserted directly under the environment header with the block's indentation. - header = new.index("aggregated_environment:\n") - assert new.index("TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS") > header - assert new.count("TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS") == 1 + assert yaml.safe_load(new)["roles"]["agg"]["env"] == { + "HF_HUB_OFFLINE": "1", + "TRTLLM_ENABLE_PDL": "1", + "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS": "1.78", + } def test_trtllm_rewrite_replaces_existing_value(): injector = get_injector("dynamo-trt") recipe = TRTLLM_RECIPE.replace( - " HF_HUB_OFFLINE: '1'\n", - " HF_HUB_OFFLINE: '1'\n TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '9'\n", + " HF_HUB_OFFLINE: '1'\n", + " HF_HUB_OFFLINE: '1'\n TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '9'\n", ) new, count = injector.rewrite(recipe, 3.02, _noop) assert count == 1 - assert "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.02'" in new - assert "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '9'" not in new + assert yaml.safe_load(new)["roles"]["agg"]["env"]["TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS"] == "2.02" def test_trtllm_rewrite_real_removes_forced_acceptance(): @@ -76,3 +79,39 @@ def test_trtllm_spec_tokens_from_recipe(): injector = get_injector("dynamo-trt") assert injector.spec_tokens_from_recipe(TRTLLM_RECIPE) == 3 assert injector.spec_tokens_from_recipe("name: x\n") is None + + +def test_schema2_injectors_target_worker_roles_and_preserve_aliases(): + recipe = '''schema: 2 +engine: sglang +roles: + prefill: + nodes: 1 + env: &common + KEEP: yes + args: + speculative-num-steps: 3 + decode: + nodes: 1 + env: *common + agg: + nodes: 1 +frontend: + env: + KEEP_FRONTEND: yes +benchmark: + env: + KEEP_CLIENT: yes +''' + for framework, variable in [("dynamo-sglang", "SGLANG_SIMULATE_ACC_LEN"), + ("dynamo-trt", "TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS")]: + injector = get_injector(framework) + rewritten, count = injector.rewrite(recipe, 2.5, _noop) + data = yaml.safe_load(rewritten) + assert count == 3 + assert all(variable in role["env"] for role in data["roles"].values()) + assert data["roles"]["decode"]["env"]["KEEP"] is True + assert data["frontend"] == yaml.safe_load(recipe)["frontend"] + assert data["benchmark"] == yaml.safe_load(recipe)["benchmark"] + real, _ = injector.rewrite_real(rewritten, _noop) + assert variable not in real diff --git a/utils/evals/EVALS.md b/utils/evals/EVALS.md index 6d7647947b..b1809fb10f 100644 --- a/utils/evals/EVALS.md +++ b/utils/evals/EVALS.md @@ -501,7 +501,7 @@ Multi-node evals support two hardware paths: - Eval artifacts copied to `/run_logs/slurm_job-*/eval_results/` - `runners/launch_mi355x-amds.sh` skips benchmark result collection when `EVAL_ONLY=true` and uses `find` to locate eval results -**NVIDIA Slurm multi-node (GB200, GB300, B200, B300, H100, H200)** runs through [srt-slurm](https://github.com/NVIDIA/srt-slurm) on the `sa-submission-q2-2026` branch. +**NVIDIA Slurm multi-node (GB200, GB300, B200, B300, H100, H200)** runs through [srt-slurm](https://github.com/NVIDIA/srt-slurm) at the shared Git submodule revision at `utils/srt-slurm`. Native `post_eval.command` and `post_eval.passthrough_env` select the InferenceX eval dispatcher without modifying the upstream checkout. - `do_sweep.py` skips the benchmark stage when `EVAL_ONLY=true`, runs `_run_post_eval()` directly - In eval-only mode, uses the full `wait_for_model()` health check (same as benchmark stage) since the benchmark health check was skipped - The registered srt-slurm `lm-eval` post-runner sources InferenceX's `benchmark_lib.sh` from the mounted workspace (`/infmax-workspace`). Kimi-selected launches patch that hook to use generic `run_eval` dispatch while preserving lm-eval as the default. diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 967280d644..be33c305dc 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -101,6 +101,26 @@ def test_aggregated_worker_expands_to_legacy_matrix_pair(): } +@pytest.mark.parametrize("roles, expected", [ + ({"agg": {"nodes": 3, "workers": 6}}, 3), + ({"prefill": {"nodes": 2}, "decode": {"nodes": 4}}, 6), + ({"prefill": {"nodes": 2}, "decode": {"nodes": "colocate"}}, 2), + ({"prefill": {"nodes": 2}, "decode": {"workers": 1}}, None), +]) +def test_multinode_node_count_reads_schema_two_roles(tmp_path, monkeypatch, roles, expected): + recipe = tmp_path / "benchmarks/multi_node/srt-slurm-recipes/test.yaml" + recipe.parent.mkdir(parents=True) + recipe.write_text(yaml.safe_dump({"schema": 2, "roles": roles})) + import infx.matrix.generate as generate + monkeypatch.setattr(generate, "__file__", str(tmp_path / "infx/matrix/generate.py")) + prefill = {"additional-settings": ["CONFIG_FILE=recipes/test.yaml"]} + if expected is None: + with pytest.raises(ValueError, match="role 'decode' must specify nodes"): + generate.recipe_node_count(prefill, {}) + else: + assert generate.recipe_node_count(prefill, {}) == expected + + def test_multinode_node_count_uses_role_gpu_footprints(sample_runner_config): prefill = {"num-worker": 3, "tp": 2, "pp": 1, "pcp-size": 1} decode = {"num-worker": 2, "tp": 8, "pp": 1, "pcp-size": 1} @@ -140,16 +160,16 @@ def test_multinode_node_count_resolves_heterogeneous_worker_hardware( "recipes/test.yaml", "benchmarks/multi_node/srt-slurm-recipes/test.yaml", ]) -@pytest.mark.parametrize(("resources", "expected_nodes"), [ - ({"agg_nodes": 3}, 3), - ({"prefill_nodes": 2, "decode_nodes": 3}, 5), +@pytest.mark.parametrize(("roles", "expected_nodes"), [ + ({"agg": {"nodes": 3}}, 3), + ({"prefill": {"nodes": 2}, "decode": {"nodes": 3}}, 5), ]) -def test_multinode_node_count_prefers_recipe_resources( - tmp_path, monkeypatch, config_file, resources, expected_nodes, +def test_multinode_node_count_prefers_recipe_roles( + tmp_path, monkeypatch, config_file, roles, expected_nodes, ): recipe = tmp_path / "benchmarks/multi_node/srt-slurm-recipes/test.yaml" recipe.parent.mkdir(parents=True) - recipe.write_text(yaml.safe_dump({"resources": resources})) + recipe.write_text(yaml.safe_dump({"schema": 2, "roles": roles})) monkeypatch.setattr( generate_sweep_configs, "__file__", str(tmp_path / "infx/matrix/generate.py"), diff --git a/utils/srt-slurm b/utils/srt-slurm new file mode 160000 index 0000000000..984180e5b8 --- /dev/null +++ b/utils/srt-slurm @@ -0,0 +1 @@ +Subproject commit 984180e5b8755aef85e9995048b5a16cb5336bce diff --git a/utils/test_aggregate_power_multinode.py b/utils/test_aggregate_power_multinode.py index 0f7ba5ccf8..3235b88ebb 100644 --- a/utils/test_aggregate_power_multinode.py +++ b/utils/test_aggregate_power_multinode.py @@ -618,3 +618,38 @@ def test_incomplete_status_rejected(self, tmp_path): pkg = build_package(tmp_path) _edit_manifest(pkg, status="incomplete", publication_valid=False) assert_invalid(pkg, "package_recompute_invalid") + + +@pytest.mark.parametrize("utilization", [("", ""), ("75.5", "0.9")]) +def test_v2_samples_preserve_energy(tmp_path, utilization): + """The pinned producer's optional utilization columns preserve board energy.""" + pkg = build_package(tmp_path) + path = pkg.power_dir / "samples.csv" + with path.open(newline="") as handle: + rows = list(csv.reader(handle)) + with path.open("w", newline="") as handle: + writer = csv.writer(handle) + writer.writerow(rows[0] + ["gpu_util_pct", "sm_active"]) + writer.writerows([[2, *row[1:], *utilization] for row in rows[1:]]) + assert pkg.run(require_power=True) == 0 + assert pkg.agg()["power_valid"] == 1 + assert pkg.agg()["total_gpu_energy_j"] == pytest.approx(84000) + + +@pytest.mark.parametrize("row", [ + [1, 1, 0, "node", 0, "GPU-0", 300, "", ""], + [2, 1, 0, "node", 0, "GPU-0", 300, "nan", ""], + [2, 1, 0, "node", 0, "GPU-0", 300, 101, ""], + [2, 1, 0, "node", 0, "GPU-0", 300, 50, 1.1], + [2, 1, 0, "node", 0, "GPU-0", 300], +]) +def test_v2_samples_reject_mixed_versions_and_invalid_utilization(tmp_path, row): + path = tmp_path / "samples.csv" + with path.open("w", newline="") as handle: + writer = csv.writer(handle) + writer.writerow(["schema_version", "timestamp_unix", "scrape_seq", "hostname", + "gpu_index", "gpu_uuid", "power_w", "gpu_util_pct", "sm_active"]) + writer.writerow(row) + rows, reasons = apm.read_samples(path) + assert not rows + assert reasons == ("samples_csv_malformed",)