From dcdc91390a5ce3d4fd9b87ef8aa2dc71a2b0065c Mon Sep 17 00:00:00 2001 From: alec-flowers Date: Wed, 25 Feb 2026 01:36:53 +0000 Subject: [PATCH 1/2] feat: add --kv-transfer-config NixlConnector to disagg scripts and recipes Signed-off-by: alec-flowers --- examples/backends/vllm/launch/disagg.sh | 3 ++- examples/backends/vllm/launch/disagg_multimodal_epd.sh | 6 +++--- examples/backends/vllm/launch/disagg_multimodal_llama.sh | 2 ++ examples/backends/vllm/launch/disagg_router.sh | 8 ++++++-- examples/backends/vllm/launch/disagg_same_gpu.sh | 2 ++ recipes/deepseek-r1/vllm/disagg/deploy_hopper_16gpu.yaml | 4 ++++ recipes/llama-3-70b/vllm/disagg-multi-node/deploy.yaml | 4 ++-- recipes/llama-3-70b/vllm/disagg-single-node/deploy.yaml | 4 ++-- recipes/qwen3-32b/vllm/disagg-kv-router/deploy.yaml | 4 ++++ 9 files changed, 27 insertions(+), 10 deletions(-) diff --git a/examples/backends/vllm/launch/disagg.sh b/examples/backends/vllm/launch/disagg.sh index 6e807fbff7a4..19520cd43890 100755 --- a/examples/backends/vllm/launch/disagg.sh +++ b/examples/backends/vllm/launch/disagg.sh @@ -10,7 +10,7 @@ python -m dynamo.frontend & # --enforce-eager is added for quick deployment. for production use, need to remove this flag DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT1:-8081} \ - CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.vllm --model Qwen/Qwen3-0.6B --enforce-eager --disaggregation-mode decode & + CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.vllm --model Qwen/Qwen3-0.6B --enforce-eager --disaggregation-mode decode --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' & DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT2:-8082} \ VLLM_NIXL_SIDE_CHANNEL_PORT=20097 \ @@ -18,4 +18,5 @@ CUDA_VISIBLE_DEVICES=1 python3 -m dynamo.vllm \ --model Qwen/Qwen3-0.6B \ --enforce-eager \ --disaggregation-mode prefill \ + --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20081","enable_kv_cache_events":true}' diff --git a/examples/backends/vllm/launch/disagg_multimodal_epd.sh b/examples/backends/vllm/launch/disagg_multimodal_epd.sh index 9b54804d1313..1e3d4d01a70d 100755 --- a/examples/backends/vllm/launch/disagg_multimodal_epd.sh +++ b/examples/backends/vllm/launch/disagg_multimodal_epd.sh @@ -66,17 +66,17 @@ DYN_DECODE_GPU_MEM=${DYN_DECODE_GPU_MEM:-0.9} # Start encode worker echo "Starting encode worker on GPU $DYN_ENCODE_WORKER_GPU (GPU mem: $DYN_ENCODE_GPU_MEM)..." -VLLM_NIXL_SIDE_CHANNEL_PORT=20097 CUDA_VISIBLE_DEVICES=$DYN_ENCODE_WORKER_GPU python -m dynamo.vllm --multimodal-encode-worker --enable-multimodal --model $MODEL_NAME --gpu-memory-utilization $DYN_ENCODE_GPU_MEM $EXTRA_ARGS --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080"}' & +VLLM_NIXL_SIDE_CHANNEL_PORT=20097 CUDA_VISIBLE_DEVICES=$DYN_ENCODE_WORKER_GPU python -m dynamo.vllm --multimodal-encode-worker --enable-multimodal --model $MODEL_NAME --gpu-memory-utilization $DYN_ENCODE_GPU_MEM $EXTRA_ARGS --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080"}' & # Start prefill worker (also handles encode routing via --route-to-encoder) echo "Starting prefill worker on GPU $DYN_PREFILL_WORKER_GPU (GPU mem: $DYN_PREFILL_GPU_MEM)..." VLLM_NIXL_SIDE_CHANNEL_PORT=20098 \ -CUDA_VISIBLE_DEVICES=$DYN_PREFILL_WORKER_GPU python -m dynamo.vllm --route-to-encoder --disaggregation-mode prefill --enable-multimodal --enable-mm-embeds --model $MODEL_NAME --gpu-memory-utilization $DYN_PREFILL_GPU_MEM $EXTRA_ARGS --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20081"}' & +CUDA_VISIBLE_DEVICES=$DYN_PREFILL_WORKER_GPU python -m dynamo.vllm --route-to-encoder --disaggregation-mode prefill --enable-multimodal --enable-mm-embeds --model $MODEL_NAME --gpu-memory-utilization $DYN_PREFILL_GPU_MEM $EXTRA_ARGS --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20081"}' & # Start decode worker echo "Starting decode worker on GPU $DYN_DECODE_WORKER_GPU (GPU mem: $DYN_DECODE_GPU_MEM)..." VLLM_NIXL_SIDE_CHANNEL_PORT=20099 \ -CUDA_VISIBLE_DEVICES=$DYN_DECODE_WORKER_GPU python -m dynamo.vllm --multimodal-decode-worker --enable-multimodal --enable-mm-embeds --model $MODEL_NAME --gpu-memory-utilization $DYN_DECODE_GPU_MEM $EXTRA_ARGS --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20082"}' & +CUDA_VISIBLE_DEVICES=$DYN_DECODE_WORKER_GPU python -m dynamo.vllm --multimodal-decode-worker --enable-multimodal --enable-mm-embeds --model $MODEL_NAME --gpu-memory-utilization $DYN_DECODE_GPU_MEM $EXTRA_ARGS --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20082"}' & echo "==================================================" echo "All components started. Waiting for initialization..." diff --git a/examples/backends/vllm/launch/disagg_multimodal_llama.sh b/examples/backends/vllm/launch/disagg_multimodal_llama.sh index 557e9239ab3f..98beca40965b 100755 --- a/examples/backends/vllm/launch/disagg_multimodal_llama.sh +++ b/examples/backends/vllm/launch/disagg_multimodal_llama.sh @@ -73,6 +73,7 @@ if [[ $HEAD_NODE -eq 1 ]]; then --enable-multimodal \ --model $MODEL_NAME \ --disaggregation-mode prefill \ + --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \ $MODEL_SPECIFIC_ARGS \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080"}' \ "${EXTRA_ARGS[@]}" & @@ -84,6 +85,7 @@ else python -m dynamo.vllm \ --enable-multimodal \ --model $MODEL_NAME \ + --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \ $MODEL_SPECIFIC_ARGS \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20081"}' \ "${EXTRA_ARGS[@]}" & diff --git a/examples/backends/vllm/launch/disagg_router.sh b/examples/backends/vllm/launch/disagg_router.sh index 2c389efaa425..b2153f05147f 100755 --- a/examples/backends/vllm/launch/disagg_router.sh +++ b/examples/backends/vllm/launch/disagg_router.sh @@ -24,14 +24,16 @@ CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.vllm \ --model $MODEL \ --block-size $BLOCK_SIZE \ --enforce-eager \ - --disaggregation-mode decode & + --disaggregation-mode decode \ + --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' & VLLM_NIXL_SIDE_CHANNEL_PORT=20097 \ CUDA_VISIBLE_DEVICES=1 python3 -m dynamo.vllm \ --model $MODEL \ --block-size $BLOCK_SIZE \ --enforce-eager \ - --disaggregation-mode decode & + --disaggregation-mode decode \ + --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' & # two prefill workers # When registered with --disaggregation-mode prefill, these workers are automatically detected @@ -42,6 +44,7 @@ CUDA_VISIBLE_DEVICES=2 python3 -m dynamo.vllm \ --block-size $BLOCK_SIZE \ --enforce-eager \ --disaggregation-mode prefill \ + --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20082","enable_kv_cache_events":true}'& VLLM_NIXL_SIDE_CHANNEL_PORT=20099 \ @@ -50,4 +53,5 @@ CUDA_VISIBLE_DEVICES=3 python3 -m dynamo.vllm \ --block-size $BLOCK_SIZE \ --enforce-eager \ --disaggregation-mode prefill \ + --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20083","enable_kv_cache_events":true}' diff --git a/examples/backends/vllm/launch/disagg_same_gpu.sh b/examples/backends/vllm/launch/disagg_same_gpu.sh index c8b4d26efc1c..3e2fc63ba294 100755 --- a/examples/backends/vllm/launch/disagg_same_gpu.sh +++ b/examples/backends/vllm/launch/disagg_same_gpu.sh @@ -56,6 +56,7 @@ python3 -m dynamo.vllm \ --model Qwen/Qwen3-0.6B \ --enforce-eager \ --disaggregation-mode decode \ + --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \ --gpu-memory-utilization ${GPU_MEM_FRACTION} & DECODE_PID=$! @@ -76,6 +77,7 @@ python3 -m dynamo.vllm \ --model Qwen/Qwen3-0.6B \ --enforce-eager \ --disaggregation-mode prefill \ + --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' \ --gpu-memory-utilization ${GPU_MEM_FRACTION} \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20081","enable_kv_cache_events":true}' diff --git a/recipes/deepseek-r1/vllm/disagg/deploy_hopper_16gpu.yaml b/recipes/deepseek-r1/vllm/disagg/deploy_hopper_16gpu.yaml index b3df6abc4bb3..cc6f3f08bc80 100644 --- a/recipes/deepseek-r1/vllm/disagg/deploy_hopper_16gpu.yaml +++ b/recipes/deepseek-r1/vllm/disagg/deploy_hopper_16gpu.yaml @@ -97,6 +97,8 @@ spec: - "512" - --compilation_config - '{"pass_config":{"fuse_norm_quant":true,"eliminate_noops":true},"cudagraph_mode":"FULL_DECODE_ONLY"}' + - --kv-transfer-config + - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' prefill: componentType: worker subComponentType: prefill @@ -144,6 +146,8 @@ spec: - /model-cache/deepseek-r1 - --disaggregation-mode - prefill + - --kv-transfer-config + - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - --served-model-name - deepseek-ai/DeepSeek-R1 - --all2all-backend diff --git a/recipes/llama-3-70b/vllm/disagg-multi-node/deploy.yaml b/recipes/llama-3-70b/vllm/disagg-multi-node/deploy.yaml index 70ef1f2bf75e..7f08da109454 100644 --- a/recipes/llama-3-70b/vllm/disagg-multi-node/deploy.yaml +++ b/recipes/llama-3-70b/vllm/disagg-multi-node/deploy.yaml @@ -42,7 +42,7 @@ spec: - name: HF_HOME value: /opt/models args: - - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 8 --data-parallel-size 1 --disaggregation-mode prefill --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128" + - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 8 --data-parallel-size 1 --disaggregation-mode prefill --kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\"}' --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128" command: - /bin/sh - -c @@ -73,7 +73,7 @@ spec: - name: HF_HOME value: /opt/models args: - - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 8 --data-parallel-size 1 --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128" + - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 8 --data-parallel-size 1 --kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\"}' --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128" command: - /bin/sh - -c diff --git a/recipes/llama-3-70b/vllm/disagg-single-node/deploy.yaml b/recipes/llama-3-70b/vllm/disagg-single-node/deploy.yaml index d8073ddde7fc..eefdf339c045 100644 --- a/recipes/llama-3-70b/vllm/disagg-single-node/deploy.yaml +++ b/recipes/llama-3-70b/vllm/disagg-single-node/deploy.yaml @@ -54,7 +54,7 @@ spec: - name: HF_HOME value: /opt/models args: - - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 2 --data-parallel-size 1 --disaggregation-mode prefill --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128" + - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 2 --data-parallel-size 1 --disaggregation-mode prefill --kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\"}' --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128" command: - /bin/sh - -c @@ -97,7 +97,7 @@ spec: - name: HF_HOME value: /opt/models args: - - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 4 --data-parallel-size 1 --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128" + - "python3 -m dynamo.vllm --model $MODEL_PATH --served-model-name $SERVED_MODEL_NAME --tensor-parallel-size 4 --data-parallel-size 1 --kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\"}' --gpu-memory-utilization 0.90 --no-enable-prefix-caching --block-size 128" command: - /bin/sh - -c diff --git a/recipes/qwen3-32b/vllm/disagg-kv-router/deploy.yaml b/recipes/qwen3-32b/vllm/disagg-kv-router/deploy.yaml index 37bd33541122..3c7ee4303156 100644 --- a/recipes/qwen3-32b/vllm/disagg-kv-router/deploy.yaml +++ b/recipes/qwen3-32b/vllm/disagg-kv-router/deploy.yaml @@ -45,6 +45,8 @@ spec: - Qwen/Qwen3-32B - --disaggregation-mode - decode + - --kv-transfer-config + - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - --tensor-parallel-size - '2' - --disable-log-requests @@ -99,6 +101,8 @@ spec: - Qwen/Qwen3-32B - --disaggregation-mode - prefill + - --kv-transfer-config + - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - --tensor-parallel-size - '2' - --disable-log-requests From 82047662ce2c4312e22ed224e933c5abf4b4bcc5 Mon Sep 17 00:00:00 2001 From: alec-flowers Date: Wed, 25 Feb 2026 01:55:28 +0000 Subject: [PATCH 2/2] add nixl kv-transfer-config to kvbm disagg deploy decode workers Signed-off-by: alec-flowers --- examples/backends/vllm/deploy/disagg_kvbm.yaml | 2 ++ examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml | 2 ++ examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/examples/backends/vllm/deploy/disagg_kvbm.yaml b/examples/backends/vllm/deploy/disagg_kvbm.yaml index 67bbf8cfd49a..a50724d656c7 100644 --- a/examples/backends/vllm/deploy/disagg_kvbm.yaml +++ b/examples/backends/vllm/deploy/disagg_kvbm.yaml @@ -33,6 +33,8 @@ spec: - Qwen/Qwen3-8B - --disaggregation-mode - decode + - --kv-transfer-config + - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - --max-model-len - "32000" - --enforce-eager diff --git a/examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml b/examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml index 3e53e90926d8..7b631abbb3e5 100644 --- a/examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml +++ b/examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml @@ -33,6 +33,8 @@ spec: - Qwen/Qwen3-8B - --disaggregation-mode - decode + - --kv-transfer-config + - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - --max-model-len - "32000" - --enforce-eager diff --git a/examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml b/examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml index aa8597473085..c65ada4b9fa9 100644 --- a/examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml +++ b/examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml @@ -35,6 +35,8 @@ spec: - Qwen/Qwen3-8B - --disaggregation-mode - decode + - --kv-transfer-config + - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - --gpu-memory-utilization - "0.23" - --max-model-len