diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp16-mk-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp16-mk-variants.yaml new file mode 100644 index 0000000000..ae86fd9dbc --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/agg-tp16-mk-variants.yaml @@ -0,0 +1,157 @@ +schema: 2 + +base: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-agg-tp16-mk + identity: + model: {} + container: {} + frameworks: + dynamo: "1.6.0" + sglang: "0.0.0.dev1+ge79de62d2.d20261104" + dynamo: + install: false + request_plane: tcp + event_plane: zmq + # The DFlash2 draft is read from the shared Hub cache. + environment: + HF_HUB_CACHE: /hf_hub_cache + # Tachometer's per-node exporters slow decode steps and add host memory; + # InferenceX does not read its output. + observability: + tachometer: + enabled: false + slurm: + time_limit: "6:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it. + - name: otel-collector + type: otel-collector + enabled: false + frontend: + type: dynamo + enable_multiple_frontends: false + placement: + node: dedicated + env: + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + args: + router-mode: kv + router-session-affinity-ttl-secs: "14400" + dyn-chat-processor: sglang + dyn-preprocess-workers: "4" + trust-remote-code: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + agg: + nodes: 4 + workers: 1 + gpus: 16 + env: + PYTHONUNBUFFERED: "1" + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + # At TP16 the sharded MoE front's per-rank N (1048) has no split-K tile for captured + # batches of 18-32 rows; the unsharded front captures every batch. + SGLANG_K3_SHARDED_MOE: "0" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 16 + expert-parallel-size: 1 + # The K3.1 megakernel serves TP16 verify steps of 1, 2 or 4 requests with the trtllm + # DSA backends, MXFP4 MoE, FP8 KV and 64-token pages; larger batches run eager. + enable-nvidia-megakernel: true + # The megakernel spec requires a text-only model and an explicit context length. + language-only: true + context-length: 1048576 + dsa-prefill-backend: trtllm + dsa-decode-backend: trtllm + moe-runner-backend: flashinfer_mxfp4 + kv-cache-dtype: fp8_e4m3 + page-size: 64 + mem-fraction-static: 0.88 + chunked-prefill-size: 16384 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 256 + enable-linear-replayssm-spec: true + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-draft-attention-backend: cutedsl_mla + speculative-draft-kv-cache-dtype: fp8_e4m3 + # The golden AL curve was collected with block verification. + speculative-use-block-verification: true + # DFlash2 verifies the current token plus 7 drafts. + speculative-num-draft-tokens: 8 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + # Full collections run 100x less often than with the default threshold. + gc-threshold: [700, 10, 1000] + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":4}' + weight-loader-disable-mmap: true + weight-loader-prefetch-checkpoints: false + watchdog-timeout: 3600 + enable-metrics: true + enable-cache-report: true + kv_events: false + sbatch_directives: + mem: "0" + exclusive: "" + srun_options: + container-remap-root: "" + mem: "0" + benchmark: + placement: + node: dedicated + colocate_with_frontend: true + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + # The client splits this value on whitespace, so it has none. + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"thinking":true},"temperature":1,"top_p":0.95}' + AIPERF_DATASET_MMAP_CACHE_ENABLED: "true" + AGENTIC_WARMUP_GRACE_PERIOD: "1800" + +# max-running-requests equals the concurrency, so every verify step stays on a megakernel graph. +override_tp16_c1: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-agg-tp16-mk-c1 + roles: + agg: + args: + max-running-requests: 1 + +override_tp16_c2: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-agg-tp16-mk-c2 + roles: + agg: + args: + max-running-requests: 2 + +override_tp16_c4: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-agg-tp16-mk-c4 + roles: + agg: + args: + max-running-requests: 4 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp16-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp16-variants.yaml new file mode 100644 index 0000000000..9b5283d514 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp16-variants.yaml @@ -0,0 +1,232 @@ +schema: 2 + +base: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp16 + identity: + model: {} + container: {} + frameworks: + dynamo: "1.6.0" + sglang: "0.0.0.dev1+ge79de62d2.d20261104" + dynamo: + install: false + request_plane: tcp + event_plane: zmq + # The DFlash2 draft is read from the shared Hub cache. + environment: + HF_HUB_CACHE: /hf_hub_cache + # Tachometer's per-node exporters slow decode steps and add host memory; + # InferenceX does not read its output. + observability: + tachometer: + enabled: false + slurm: + time_limit: "6:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it. + - name: otel-collector + type: otel-collector + enabled: false + frontend: + type: dynamo + enable_multiple_frontends: false + placement: + node: dedicated + env: + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + args: + router-mode: kv + dyn-chat-processor: sglang + dyn-preprocess-workers: "4" + trust-remote-code: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + CUDA_MODULE_LOADING: EAGER + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + SGLANG_DSA_MQA_LOGITS_FREE_MEM_FRACTION: "0.1" + SGLANG_NUMA_BIND_V2: "0" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + # Some GB300 nodes list a BMC interface with the same address on every node first. + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 8 + dcp-size: 8 + dcp-comm-backend: fi_a2a + expert-parallel-size: 2 + dsa-prefill-backend: prims_ts + dsa-decode-backend: prims_ts + kv-cache-dtype: fp8_e4m3 + page-size: 64 + mem-fraction-static: 0.88 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + schedule-policy: shortest-prefill-first + mamba-full-memory-ratio: 1.2 + mamba-max-states-per-path: 1 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 256 + enable-hierarchical-cache: true + # 96 GB per rank, 384 GB per prefill node. + hicache-size: 96 + hicache-write-policy: write_back + disaggregation-transfer-backend: mooncake + disaggregation-decode-allocation-policy: early + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-draft-attention-backend: cutedsl_mla + speculative-draft-kv-cache-dtype: fp8_e4m3 + # The golden AL curve was collected with block verification. + speculative-use-block-verification: true + # DFlash2 verifies the current token plus 7 drafts. + speculative-num-draft-tokens: 8 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + # A session's turns stay on one prefill worker. + router-mode: kv + router-host-cache-hit-weight: 1 + router-kv-overlap-score-credit: 4 + router-session-affinity-ttl-secs: "14400" + # Full collections run 100x less often than with the default threshold. + gc-threshold: [700, 10, 1000] + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":4}' + weight-loader-disable-mmap: true + weight-loader-prefetch-checkpoints: false + weight-loader-drop-cache-after-load: false + watchdog-timeout: 3600 + enable-metrics: true + enable-cache-report: true + kv_events: true + decode: + nodes: 4 + workers: 1 + gpus: 16 + env: + PYTHONUNBUFFERED: "1" + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1536" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + NVSHMEM_REMOTE_TRANSPORT: none + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + # DCP8 x attention DP2 over the 16 GPUs; MoE runs EP16. + tensor-parallel-size: 16 + dcp-size: 8 + attn-dp-size: 2 + dcp-comm-backend: fi_a2a + expert-parallel-size: 16 + moe-a2a-backend: megamoe + enable-dp-lm-head: true + dsa-prefill-backend: prims_ts + dsa-decode-backend: prims_ts + kv-cache-dtype: fp8_e4m3 + page-size: 64 + mem-fraction-static: 0.85 + max-running-requests: 384 + cuda-graph-max-bs-decode: 192 + disaggregation-transfer-backend: mooncake + disaggregation-decode-enable-radix-cache: true + disaggregation-decode-extra-slots: 16 + enable-linear-replayssm-spec: true + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-draft-attention-backend: cutedsl_mla + speculative-draft-kv-cache-dtype: fp8_e4m3 + speculative-use-block-verification: true + speculative-num-draft-tokens: 8 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + gc-threshold: [700, 10, 1000] + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":4}' + weight-loader-disable-mmap: true + weight-loader-prefetch-checkpoints: false + weight-loader-drop-cache-after-load: false + watchdog-timeout: 3600 + enable-metrics: true + enable-cache-report: true + kv_events: false + sbatch_directives: + mem: "0" + exclusive: "" + srun_options: + container-remap-root: "" + mem: "0" + benchmark: + placement: + node: dedicated + colocate_with_frontend: true + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + # The client splits this value on whitespace, so it has none. + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"thinking":true},"temperature":1,"top_p":0.95}' + AIPERF_DATASET_MMAP_CACHE_ENABLED: "true" + AGENTIC_WARMUP_GRACE_PERIOD: "1800" + +override_3p1d_c300: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp16-3p1d-c300 + roles: + prefill: + nodes: 6 + workers: 3 + args: + # Verifies the current token plus 5 drafts: at this batch the shorter block is faster. + speculative-num-draft-tokens: 6 + decode: + args: + speculative-num-draft-tokens: 6 + +override_2p1d_c180: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp16-2p1d-c180 + roles: + prefill: + nodes: 4 + workers: 2 + +override_1p1d_c76: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp16-1p1d-c76 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp8-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp8-variants.yaml new file mode 100644 index 0000000000..29d46f9d5f --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/kimik3.1/sglang/gb300-fp4/agentx/disagg-tp8-variants.yaml @@ -0,0 +1,216 @@ +schema: 2 + +base: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp8 + identity: + model: {} + container: {} + frameworks: + dynamo: "1.6.0" + sglang: "0.0.0.dev1+ge79de62d2.d20261104" + dynamo: + install: false + request_plane: tcp + event_plane: zmq + # The DFlash2 draft is read from the shared Hub cache. + environment: + HF_HUB_CACHE: /hf_hub_cache + # Tachometer's per-node exporters slow decode steps and add host memory; + # InferenceX does not read its output. + observability: + tachometer: + enabled: false + slurm: + time_limit: "6:00:00" + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: gb300 + gpus_per_node: 4 + services: + - name: etcd + type: etcd + placement: + node: infra + # Disables Dynamo's per-request lifecycle span export; InferenceX does not read it. + - name: otel-collector + type: otel-collector + enabled: false + frontend: + type: dynamo + enable_multiple_frontends: false + placement: + node: dedicated + env: + DYN_ROUTER_TEMPERATURE: "0" + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + args: + router-mode: kv + # The active-request term dominates KV overlap, so each turn goes to the least-loaded + # decode worker. + router-decode-active-request-weight: "1000000" + dyn-chat-processor: sglang + dyn-preprocess-workers: "4" + trust-remote-code: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + engine: sglang + roles: + prefill: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + CUDA_MODULE_LOADING: EAGER + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + SGLANG_DSA_MQA_LOGITS_FREE_MEM_FRACTION: "0.1" + SGLANG_NUMA_BIND_V2: "0" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + # Some GB300 nodes list a BMC interface with the same address on every node first. + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 8 + dcp-size: 8 + dcp-comm-backend: fi_a2a + expert-parallel-size: 2 + dsa-prefill-backend: prims_ts + dsa-decode-backend: prims_ts + kv-cache-dtype: fp8_e4m3 + page-size: 64 + mem-fraction-static: 0.88 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + schedule-policy: shortest-prefill-first + mamba-full-memory-ratio: 1.2 + mamba-max-states-per-path: 1 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 256 + enable-hierarchical-cache: true + # 96 GB per rank, 384 GB per prefill node. + hicache-size: 96 + hicache-write-policy: write_back + disaggregation-transfer-backend: mooncake + disaggregation-decode-allocation-policy: early + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-draft-attention-backend: cutedsl_mla + speculative-draft-kv-cache-dtype: fp8_e4m3 + # The golden AL curve was collected with block verification. + speculative-use-block-verification: true + # DFlash2 verifies the current token plus 7 drafts. + speculative-num-draft-tokens: 8 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + router-mode: kv + router-host-cache-hit-weight: 1 + router-kv-overlap-score-credit: 4 + router-session-affinity-ttl-secs: "14400" + # Full collections run 100x less often than with the default threshold. + gc-threshold: [700, 10, 1000] + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":4}' + weight-loader-disable-mmap: true + weight-loader-prefetch-checkpoints: false + weight-loader-drop-cache-after-load: false + watchdog-timeout: 3600 + enable-metrics: true + enable-cache-report: true + kv_events: false + decode: + nodes: 6 + workers: 3 + gpus: 8 + env: + PYTHONUNBUFFERED: "1" + SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB: "32768" + SGLANG_DCP_PUSH_COMBINE_MAX_ROWS: "1536" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + MC_FORCE_MNNVL: "1" + MC_TCP_BIND_ADDRESS: "{node}" + WITH_NVIDIA_PEERMEM: "0" + args: + served-model-name: moonshotai/Kimi-K3.1-Pro + trust-remote-code: true + tensor-parallel-size: 8 + dcp-size: 8 + dcp-comm-backend: fi_a2a + expert-parallel-size: 2 + dsa-prefill-backend: prims_ts + dsa-decode-backend: prims_ts + kv-cache-dtype: fp8_e4m3 + mem-fraction-static: 0.88 + max-running-requests: 64 + disaggregation-transfer-backend: mooncake + disaggregation-decode-enable-radix-cache: true + disaggregation-decode-extra-slots: 16 + enable-linear-replayssm-spec: true + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Kimi-K3.1-Dflash2-Preview + speculative-draft-model-revision: d0d8bc2e3c6d0a195d2e00a49017e71ca8d881f1 + speculative-draft-attention-backend: cutedsl_mla + speculative-draft-kv-cache-dtype: fp8_e4m3 + speculative-use-block-verification: true + speculative-num-draft-tokens: 8 + dyn-reasoning-parser: kimi_k3 + dyn-tool-call-parser: kimi_k3 + gc-threshold: [700, 10, 1000] + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":4}' + weight-loader-disable-mmap: true + weight-loader-prefetch-checkpoints: false + weight-loader-drop-cache-after-load: false + watchdog-timeout: 3600 + enable-metrics: true + enable-cache-report: true + kv_events: false + sbatch_directives: + mem: "0" + exclusive: "" + srun_options: + container-remap-root: "" + mem: "0" + benchmark: + placement: + node: dedicated + colocate_with_frontend: true + env: + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + # The client splits this value on whitespace, so it has none. + AIPERF_EXTRA_INPUTS: '{"chat_template_kwargs":{"thinking":true},"temperature":1,"top_p":0.95}' + AIPERF_DATASET_MMAP_CACHE_ENABLED: "true" + AGENTIC_WARMUP_GRACE_PERIOD: "1800" + +override_1p3d_c60: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp8-1p3d-c60 + +override_1p3d_c36: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp8-1p3d-c36 + +override_1p4d_c32: + name: kimik3.1-fp4-gb300-dynamo-sglang-agentx-disagg-tp8-1p4d-c32 + roles: + decode: + nodes: 8 + workers: 4 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index b219c973c8..9c217577b1 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -7444,3 +7444,95 @@ minimaxm31flash-fp4-gb300-dynamo-vllm-agentic-dspark-disagg: srt-recipe: disagg-mooncake.yaml:override_3p4d16 prefill: { num-worker: 6, tp: 2, ep: 2, dp-attn: true } decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: true } + +kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-disagg: + image: lmsysorg/sglang:dev-kimi-k31-pro@sha256:67fc77d1cbe67de27ca54d874c82757a4844743fea899ea7d0df547e30c2c391 + model: moonshotai/Kimi-K3.1-Pro + model-prefix: kimik3.1 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.6.0" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + srt-recipe-dir: kimik3.1/sglang/gb300-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: draft_model + conc-list: [300] + srt-recipe: disagg-tp16-variants.yaml:override_3p1d_c300 + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: { num-worker: 3, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: false, dcp-size: 8 } + - spec-decoding: draft_model + conc-list: [180] + srt-recipe: disagg-tp16-variants.yaml:override_2p1d_c180 + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: { num-worker: 2, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: false, dcp-size: 8 } + - spec-decoding: draft_model + conc-list: [76] + srt-recipe: disagg-tp16-variants.yaml:override_1p1d_c76 + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: { num-worker: 1, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 1, tp: 16, ep: 16, dp-attn: false, dcp-size: 8 } + - spec-decoding: draft_model + conc-list: [60] + srt-recipe: disagg-tp8-variants.yaml:override_1p3d_c60 + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: { num-worker: 1, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 3, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + - spec-decoding: draft_model + conc-list: [36] + srt-recipe: disagg-tp8-variants.yaml:override_1p3d_c36 + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: { num-worker: 1, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 3, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + - spec-decoding: draft_model + conc-list: [32] + srt-recipe: disagg-tp8-variants.yaml:override_1p4d_c32 + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: { num-worker: 1, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + decode: { num-worker: 4, tp: 8, ep: 2, dp-attn: false, dcp-size: 8 } + +kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-agg-mk: + image: lmsysorg/sglang:dev-kimi-k31-pro@sha256:67fc77d1cbe67de27ca54d874c82757a4844743fea899ea7d0df547e30c2c391 + model: moonshotai/Kimi-K3.1-Pro + model-prefix: kimik3.1 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.6.0" } + multinode: true + disagg: false + srt-recipe-dir: kimik3.1/sglang/gb300-fp4/agentx + scenarios: + agentic-coding: + - search-space: + - spec-decoding: draft_model + conc-list: [1] + num-nodes: 5 + srt-recipe: agg-tp16-mk-variants.yaml:override_tp16_c1 + kv-offloading: none + worker: { num-worker: 1, tp: 16, ep: 1, dp-attn: false } + - spec-decoding: draft_model + conc-list: [2] + num-nodes: 5 + srt-recipe: agg-tp16-mk-variants.yaml:override_tp16_c2 + kv-offloading: none + worker: { num-worker: 1, tp: 16, ep: 1, dp-attn: false } + - spec-decoding: draft_model + conc-list: [4] + num-nodes: 5 + srt-recipe: agg-tp16-mk-variants.yaml:override_tp16_c4 + kv-offloading: none + worker: { num-worker: 1, tp: 16, ep: 1, dp-attn: false } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 1ece98d196..bc808e6451 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -12088,3 +12088,14 @@ - "GPU-resident KV at concurrency 1, 2, 4, 8, 12 and 16; SGLang HiCache with the auto-sized host tier at concurrency 24, 32 and 40." - "Use gfx942 TileLang DSA with Torch DSA top-k, AITER MoE and FP8 KV, and disable INT8 QuickReduce." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4721 + +- config-keys: + - kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-disagg + - kimik3.1-fp4-gb300-dynamo-sglang-agentic-dflash-agg-mk + scenario-type: + - agentic-coding + description: + - "Add Kimi-K3.1-Pro GB300 Dynamo+SGLang AgentX points on lmsysorg/sglang:dev-kimi-k31-pro with the RadixArk/Kimi-K3.1-Dflash2-Preview DFLASH draft." + - "Disaggregated: TP8/DCP8 prefill with a HiCache host tier feeding either one TP16 (DCP8 x attention DP2, EP16 MegaMoE) decode worker (3P1D c300, 2P1D c180, 1P1D c76) or TP8/DCP8 decode workers (1P3D c60 and c36, 1P4D c32)." + - "Aggregated low-latency points: one TP16 EP1 worker with the NVIDIA megakernel (--enable-nvidia-megakernel), max-running-requests equal to the concurrency (c1, c2, c4)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4811