Skip to content
This repository was archived by the owner on Apr 20, 2026. It is now read-only.

Use mtp3 for 1k8k, 1k1k gb200 fp8 disagg low latency configs #143

Merged
trevor-m merged 1 commit into from
Feb 5, 2026
Merged

Use mtp3 for 1k8k, 1k1k gb200 fp8 disagg low latency configs #143

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions recipes/gb200-fp8/1k1k/low-latency-mtp.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: "gb200-fp8-1p-2d-low-latency-mtp"
name: "gb200-fp8-1p-2d-low-latency-mtp3"

frontend:
nginx_container: nginx
Expand Down Expand Up @@ -73,11 +73,11 @@ backend:
watchdog-timeout: 1000000
context-length: 2200
disaggregation-mode: "prefill"
mem-fraction-static: 0.90
mem-fraction-static: 0.95
max-total-tokens: 8192
chunked-prefill-size: 8192
cuda-graph-max-bs: 128
max-running-requests: 512
max-running-requests: 128
load-balance-method: "round_robin"
scheduler-recv-interval: 10
enable-flashinfer-allreduce-fusion: true
Expand All @@ -88,9 +88,9 @@ backend:
expert-parallel-size: 1
disaggregation-transfer-backend: "nixl"
speculative-algorithm: "EAGLE"
speculative-num-steps: 2
speculative-num-steps: 3
speculative-eagle-topk: 1
speculative-num-draft-tokens: 3
speculative-num-draft-tokens: 4

decode:
served-model-name: "deepseek-ai/DeepSeek-R1"
Expand Down Expand Up @@ -118,9 +118,9 @@ backend:
expert-parallel-size: 1
disaggregation-transfer-backend: "nixl"
speculative-algorithm: "EAGLE"
speculative-num-steps: 2
speculative-num-steps: 3
speculative-eagle-topk: 1
speculative-num-draft-tokens: 3
speculative-num-draft-tokens: 4

benchmark:
type: "sa-bench"
Expand Down
10 changes: 5 additions & 5 deletions recipes/gb200-fp8/1k8k/low-latency-mtp.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: "gb200-fp8-1k8k-low-latency-mtp"
name: "gb200-fp8-1k8k-low-latency-mtp3"

frontend:
nginx_container: nginx
Expand Down Expand Up @@ -86,9 +86,9 @@ backend:
expert-parallel-size: 1
disaggregation-transfer-backend: "nixl"
speculative-algorithm: "EAGLE"
speculative-num-steps: 2
speculative-num-steps: 3
speculative-eagle-topk: 1
speculative-num-draft-tokens: 3
speculative-num-draft-tokens: 4

decode:
served-model-name: "deepseek-ai/DeepSeek-R1"
Expand Down Expand Up @@ -117,9 +117,9 @@ backend:
expert-parallel-size: 1
disaggregation-transfer-backend: "nixl"
speculative-algorithm: "EAGLE"
speculative-num-steps: 2
speculative-num-steps: 3
speculative-eagle-topk: 1
speculative-num-draft-tokens: 3
speculative-num-draft-tokens: 4

benchmark:
type: "sa-bench"
Expand Down
Loading