From fde8962857c654f1ed379ad6c16769e94eedad78 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 7 Aug 2026 10:06:44 -0700 Subject: [PATCH 1/2] fix(vllm): align KVBM decode KV cache layout Signed-off-by: jthomson04 --- examples/backends/vllm/deploy/disagg_kvbm.yaml | 1 + examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml | 1 + examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml | 1 + examples/backends/vllm/launch/disagg_kvbm.sh | 2 +- examples/backends/vllm/launch/disagg_kvbm_2p2d.sh | 4 ++-- examples/backends/vllm/launch/disagg_kvbm_router.sh | 2 ++ tests/kvbm_integration/test_determinism_disagg.py | 3 +++ 7 files changed, 11 insertions(+), 3 deletions(-) diff --git a/examples/backends/vllm/deploy/disagg_kvbm.yaml b/examples/backends/vllm/deploy/disagg_kvbm.yaml index 489ba1d552db..0ad4ef84cb28 100644 --- a/examples/backends/vllm/deploy/disagg_kvbm.yaml +++ b/examples/backends/vllm/deploy/disagg_kvbm.yaml @@ -24,6 +24,7 @@ spec: - Qwen/Qwen3-8B - --disaggregation-mode - decode + - --disable-hybrid-kv-cache-manager - --kv-transfer-config - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - --max-model-len diff --git a/examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml b/examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml index afefef1c0f0d..864fd0db6ee3 100644 --- a/examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml +++ b/examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml @@ -24,6 +24,7 @@ spec: - Qwen/Qwen3-8B - --disaggregation-mode - decode + - --disable-hybrid-kv-cache-manager - --kv-transfer-config - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - --max-model-len diff --git a/examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml b/examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml index 5dba86cffc0d..39cdf1d25322 100644 --- a/examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml +++ b/examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml @@ -24,6 +24,7 @@ spec: - Qwen/Qwen3-8B - --disaggregation-mode - decode + - --disable-hybrid-kv-cache-manager - --kv-transfer-config - '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - --gpu-memory-utilization diff --git a/examples/backends/vllm/launch/disagg_kvbm.sh b/examples/backends/vllm/launch/disagg_kvbm.sh index 2bc0ddc286c5..27d3f7bde647 100755 --- a/examples/backends/vllm/launch/disagg_kvbm.sh +++ b/examples/backends/vllm/launch/disagg_kvbm.sh @@ -18,7 +18,7 @@ python -m dynamo.frontend & # run decode worker on GPU 0, without enabling KVBM # NOTE: remove --enforce-eager for production use -CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.vllm --model "$MODEL" --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' --enforce-eager & +CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.vllm --model "$MODEL" --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' --enforce-eager --disaggregation-mode decode --disable-hybrid-kv-cache-manager & # run prefill worker on GPU 1 with KVBM enabled using 20GB of CPU cache # NOTE: remove --enforce-eager for production use diff --git a/examples/backends/vllm/launch/disagg_kvbm_2p2d.sh b/examples/backends/vllm/launch/disagg_kvbm_2p2d.sh index ec271e501da8..f216751869a5 100755 --- a/examples/backends/vllm/launch/disagg_kvbm_2p2d.sh +++ b/examples/backends/vllm/launch/disagg_kvbm_2p2d.sh @@ -18,9 +18,9 @@ python -m dynamo.frontend --router-mode kv & # run decode workers on GPU 0 and 1, without enabling KVBM # NOTE: remove --enforce-eager for production use -CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.vllm --model "$MODEL" --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' --enforce-eager --disaggregation-mode decode & +CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.vllm --model "$MODEL" --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' --enforce-eager --disaggregation-mode decode --disable-hybrid-kv-cache-manager & VLLM_NIXL_SIDE_CHANNEL_PORT=20097 \ -CUDA_VISIBLE_DEVICES=1 python3 -m dynamo.vllm --model "$MODEL" --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' --enforce-eager --disaggregation-mode decode & +CUDA_VISIBLE_DEVICES=1 python3 -m dynamo.vllm --model "$MODEL" --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' --enforce-eager --disaggregation-mode decode --disable-hybrid-kv-cache-manager & # run prefill workers on GPU 2 and 3 with KVBM enabled using 20GB of CPU cache # NOTE: use different barrier id prefixes for each prefill worker to avoid conflicts diff --git a/examples/backends/vllm/launch/disagg_kvbm_router.sh b/examples/backends/vllm/launch/disagg_kvbm_router.sh index 3f1c69078b43..7fdfe0729d6a 100755 --- a/examples/backends/vllm/launch/disagg_kvbm_router.sh +++ b/examples/backends/vllm/launch/disagg_kvbm_router.sh @@ -27,6 +27,7 @@ CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.vllm \ --model $MODEL \ --enforce-eager \ --disaggregation-mode decode \ + --disable-hybrid-kv-cache-manager \ --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' & VLLM_NIXL_SIDE_CHANNEL_PORT=20096 \ @@ -34,6 +35,7 @@ CUDA_VISIBLE_DEVICES=1 python3 -m dynamo.vllm \ --model $MODEL \ --enforce-eager \ --disaggregation-mode decode \ + --disable-hybrid-kv-cache-manager \ --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' & # two prefill workers with KVBM enabled diff --git a/tests/kvbm_integration/test_determinism_disagg.py b/tests/kvbm_integration/test_determinism_disagg.py index f8d778b5c098..aa176ca6e778 100755 --- a/tests/kvbm_integration/test_determinism_disagg.py +++ b/tests/kvbm_integration/test_determinism_disagg.py @@ -149,6 +149,9 @@ def _set_up_vllm_config(self, gpu_cache_blocks): "dynamo.vllm", "--model", os.environ.get("KVBM_MODEL_ID", "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"), + "--disaggregation-mode", + "decode", + "--disable-hybrid-kv-cache-manager", "--block-size", "16", "--max-model-len", From 7657bc145be4b2ef6fb847383104ce7ff5c48719 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 7 Aug 2026 10:32:39 -0700 Subject: [PATCH 2/2] fix(vllm): align all PdConnector worker layouts Signed-off-by: jthomson04 --- examples/backends/vllm/deploy/disagg_kvbm.yaml | 1 + examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml | 1 + examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml | 1 + examples/backends/vllm/launch/disagg_flexkv.sh | 3 ++- examples/backends/vllm/launch/disagg_kvbm.sh | 1 + examples/backends/vllm/launch/disagg_kvbm_2p2d.sh | 2 ++ examples/backends/vllm/launch/disagg_kvbm_router.sh | 2 ++ examples/backends/vllm/launch/disagg_lmcache.sh | 3 ++- tests/kvbm_integration/test_determinism_disagg.py | 2 ++ 9 files changed, 14 insertions(+), 2 deletions(-) diff --git a/examples/backends/vllm/deploy/disagg_kvbm.yaml b/examples/backends/vllm/deploy/disagg_kvbm.yaml index 0ad4ef84cb28..24e5349a8c31 100644 --- a/examples/backends/vllm/deploy/disagg_kvbm.yaml +++ b/examples/backends/vllm/deploy/disagg_kvbm.yaml @@ -54,6 +54,7 @@ spec: - Qwen/Qwen3-8B - --disaggregation-mode - prefill + - --disable-hybrid-kv-cache-manager - --max-model-len - "32000" - --enforce-eager diff --git a/examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml b/examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml index 864fd0db6ee3..33bbfd4fa5ad 100644 --- a/examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml +++ b/examples/backends/vllm/deploy/disagg_kvbm_2p2d.yaml @@ -54,6 +54,7 @@ spec: - Qwen/Qwen3-8B - --disaggregation-mode - prefill + - --disable-hybrid-kv-cache-manager - --max-model-len - "32000" - --enforce-eager diff --git a/examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml b/examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml index 39cdf1d25322..8974e64fda06 100644 --- a/examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml +++ b/examples/backends/vllm/deploy/disagg_kvbm_tp2.yaml @@ -60,6 +60,7 @@ spec: - Qwen/Qwen3-8B - --disaggregation-mode - prefill + - --disable-hybrid-kv-cache-manager - --gpu-memory-utilization - "0.23" - --max-model-len diff --git a/examples/backends/vllm/launch/disagg_flexkv.sh b/examples/backends/vllm/launch/disagg_flexkv.sh index 77dce97d5b44..94abe528a436 100755 --- a/examples/backends/vllm/launch/disagg_flexkv.sh +++ b/examples/backends/vllm/launch/disagg_flexkv.sh @@ -17,7 +17,7 @@ print_launch_banner "Launching Disaggregated Serving + FlexKV (2 GPUs)" "$MODEL" python -m dynamo.frontend & # Run decode worker without FlexKV -CUDA_VISIBLE_DEVICES=0 python -m dynamo.vllm --model $MODEL --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' & +CUDA_VISIBLE_DEVICES=0 python -m dynamo.vllm --model $MODEL --disaggregation-mode decode --disable-hybrid-kv-cache-manager --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' & # Run prefill worker with FlexKV DYN_VLLM_KV_EVENT_PORT=20081 \ @@ -28,6 +28,7 @@ CUDA_VISIBLE_DEVICES=1 \ python -m dynamo.vllm \ --model $MODEL \ --disaggregation-mode prefill \ + --disable-hybrid-kv-cache-manager \ --kv-transfer-config '{"kv_connector":"PdConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"FlexKVConnectorV1","kv_role":"kv_both"},{"kv_connector":"NixlConnector","kv_role":"kv_both"}]},"kv_connector_module_path":"kvbm.vllm_integration.connector"}' \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20081","enable_kv_cache_events":true}' & diff --git a/examples/backends/vllm/launch/disagg_kvbm.sh b/examples/backends/vllm/launch/disagg_kvbm.sh index 27d3f7bde647..4330260b9dff 100755 --- a/examples/backends/vllm/launch/disagg_kvbm.sh +++ b/examples/backends/vllm/launch/disagg_kvbm.sh @@ -28,6 +28,7 @@ CUDA_VISIBLE_DEVICES=1 \ python3 -m dynamo.vllm \ --model "$MODEL" \ --disaggregation-mode prefill \ + --disable-hybrid-kv-cache-manager \ --kv-transfer-config '{"kv_connector":"PdConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"},{"kv_connector":"NixlConnector","kv_role":"kv_both"}]},"kv_connector_module_path":"kvbm.vllm_integration.connector"}' \ --enforce-eager \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20081","enable_kv_cache_events":true}' & diff --git a/examples/backends/vllm/launch/disagg_kvbm_2p2d.sh b/examples/backends/vllm/launch/disagg_kvbm_2p2d.sh index f216751869a5..4e4829c4c23e 100755 --- a/examples/backends/vllm/launch/disagg_kvbm_2p2d.sh +++ b/examples/backends/vllm/launch/disagg_kvbm_2p2d.sh @@ -31,6 +31,7 @@ CUDA_VISIBLE_DEVICES=2 \ python3 -m dynamo.vllm \ --model "$MODEL" \ --disaggregation-mode prefill \ + --disable-hybrid-kv-cache-manager \ --kv-transfer-config '{"kv_connector":"PdConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"},{"kv_connector":"NixlConnector","kv_role":"kv_both"}]},"kv_connector_module_path":"kvbm.vllm_integration.connector"}' \ --enforce-eager \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20082","enable_kv_cache_events":true}' & @@ -43,6 +44,7 @@ CUDA_VISIBLE_DEVICES=3 \ python3 -m dynamo.vllm \ --model "$MODEL" \ --disaggregation-mode prefill \ + --disable-hybrid-kv-cache-manager \ --kv-transfer-config '{"kv_connector":"PdConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"},{"kv_connector":"NixlConnector","kv_role":"kv_both"}]},"kv_connector_module_path":"kvbm.vllm_integration.connector"}' \ --enforce-eager \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20083","enable_kv_cache_events":true}' & diff --git a/examples/backends/vllm/launch/disagg_kvbm_router.sh b/examples/backends/vllm/launch/disagg_kvbm_router.sh index 7fdfe0729d6a..d64e1f238003 100755 --- a/examples/backends/vllm/launch/disagg_kvbm_router.sh +++ b/examples/backends/vllm/launch/disagg_kvbm_router.sh @@ -48,6 +48,7 @@ CUDA_VISIBLE_DEVICES=2 DYN_KVBM_CPU_CACHE_GB=20 \ --model $MODEL \ --enforce-eager \ --disaggregation-mode prefill \ + --disable-hybrid-kv-cache-manager \ --kv-transfer-config '{"kv_connector":"PdConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"},{"kv_connector":"NixlConnector","kv_role":"kv_both"}]},"kv_connector_module_path":"kvbm.vllm_integration.connector"}' \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20081"}' & @@ -59,6 +60,7 @@ CUDA_VISIBLE_DEVICES=3 DYN_KVBM_CPU_CACHE_GB=20 \ --model $MODEL \ --enforce-eager \ --disaggregation-mode prefill \ + --disable-hybrid-kv-cache-manager \ --kv-transfer-config '{"kv_connector":"PdConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"DynamoConnector","kv_connector_module_path":"kvbm.vllm_integration.connector","kv_role":"kv_both"},{"kv_connector":"NixlConnector","kv_role":"kv_both"}]},"kv_connector_module_path":"kvbm.vllm_integration.connector"}' \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20082"}' & diff --git a/examples/backends/vllm/launch/disagg_lmcache.sh b/examples/backends/vllm/launch/disagg_lmcache.sh index 16d169531ab0..f5567661a1db 100755 --- a/examples/backends/vllm/launch/disagg_lmcache.sh +++ b/examples/backends/vllm/launch/disagg_lmcache.sh @@ -17,7 +17,7 @@ print_launch_banner "Launching Disaggregated Serving + LMCache (2 GPUs)" "$MODEL python -m dynamo.frontend --router-mode kv & # run decode worker on GPU 0, without enabling LMCache -CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.vllm --model "$MODEL" & +CUDA_VISIBLE_DEVICES=0 python3 -m dynamo.vllm --model "$MODEL" --disaggregation-mode decode --disable-hybrid-kv-cache-manager --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' & # wait for decode worker to initialize sleep 20 @@ -28,6 +28,7 @@ CUDA_VISIBLE_DEVICES=1 \ python3 -m dynamo.vllm \ --model "$MODEL" \ --disaggregation-mode prefill \ + --disable-hybrid-kv-cache-manager \ --kv-transfer-config '{"kv_connector":"PdConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"LMCacheConnectorV1","kv_role":"kv_both"},{"kv_connector":"NixlConnector","kv_role":"kv_both"}]},"kv_connector_module_path":"kvbm.vllm_integration.connector"}' \ --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20081","enable_kv_cache_events":true}' & diff --git a/tests/kvbm_integration/test_determinism_disagg.py b/tests/kvbm_integration/test_determinism_disagg.py index aa176ca6e778..32b2436a52a4 100755 --- a/tests/kvbm_integration/test_determinism_disagg.py +++ b/tests/kvbm_integration/test_determinism_disagg.py @@ -169,6 +169,7 @@ def _set_up_vllm_config(self, gpu_cache_blocks): os.environ.get("KVBM_MODEL_ID", "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"), "--disaggregation-mode", "prefill", + "--disable-hybrid-kv-cache-manager", "--block-size", "16", "--max-model-len", @@ -303,6 +304,7 @@ def start_server(self, timeout: int = 300) -> bool: prefiller_env = self.env.copy() prefiller_env["CUDA_VISIBLE_DEVICES"] = "1" + prefiller_env["VLLM_NIXL_SIDE_CHANNEL_PORT"] = "20097" # Launch frontend first self.process_frontend = subprocess.Popen(