diff --git a/lib/sidecar/vllm/launch/disagg.sh b/lib/sidecar/vllm/launch/disagg.sh index 09100d4f990c..81b12cefee79 100755 --- a/lib/sidecar/vllm/launch/disagg.sh +++ b/lib/sidecar/vllm/launch/disagg.sh @@ -34,6 +34,7 @@ while [[ $# -gt 0 ]]; do echo echo "Environment overrides:" echo " MODEL Model to serve (default: Qwen/Qwen3-0.6B)" + echo " VLLM_RUST_FRONTEND_PATH Native frontend binary (default: vllm-rs on PATH)" echo " DYN_HTTP_PORT Dynamo frontend port (default: 8000)" echo " DYN_SYSTEM_PORT1 Decode sidecar system port (default: 8081)" echo " DYN_SYSTEM_PORT2 Prefill sidecar system port (default: 8082)" @@ -86,14 +87,14 @@ print_launch_banner "Launching vLLM Native-gRPC Sidecar Disaggregated Serving (2 "Decode: GPU ${VLLM_DECODE_GPU}, gRPC 127.0.0.1:${VLLM_DECODE_GRPC_PORT}" \ "Prefill: GPU ${VLLM_PREFILL_GPU}, gRPC 127.0.0.1:${VLLM_PREFILL_GRPC_PORT}" -python -m dynamo.frontend & +python3 -m dynamo.frontend & # vllm-rs manages the headless Python engines and exposes native gRPC on # loopback. Arguments after -- are forwarded to each managed engine. # shellcheck disable=SC2086 # GPU_MEM_ARGS intentionally expands into multiple flags. CUDA_VISIBLE_DEVICES="$VLLM_DECODE_GPU" \ VLLM_NIXL_SIDE_CHANNEL_PORT="$VLLM_DECODE_NIXL_SIDE_CHANNEL_PORT" \ -vllm-rs serve "$MODEL" \ +"${VLLM_RUST_FRONTEND_PATH:-vllm-rs}" serve "$MODEL" \ --host 127.0.0.1 \ --port "$VLLM_DECODE_HTTP_PORT" \ --grpc-port "$VLLM_DECODE_GRPC_PORT" \ @@ -108,7 +109,7 @@ vllm-rs serve "$MODEL" \ # shellcheck disable=SC2086 # GPU_MEM_ARGS intentionally expands into multiple flags. CUDA_VISIBLE_DEVICES="$VLLM_PREFILL_GPU" \ VLLM_NIXL_SIDE_CHANNEL_PORT="$VLLM_PREFILL_NIXL_SIDE_CHANNEL_PORT" \ -vllm-rs serve "$MODEL" \ +"${VLLM_RUST_FRONTEND_PATH:-vllm-rs}" serve "$MODEL" \ --host 127.0.0.1 \ --port "$VLLM_PREFILL_HTTP_PORT" \ --grpc-port "$VLLM_PREFILL_GRPC_PORT" \ @@ -122,13 +123,13 @@ vllm-rs serve "$MODEL" \ "${EXTRA_ARGS[@]}" & DYN_SYSTEM_PORT="${DYN_SYSTEM_PORT1:-8081}" \ - dynamo-vllm-sidecar \ + python3 -m dynamo.vllm.sidecar \ --grpc-endpoint "127.0.0.1:${VLLM_DECODE_GRPC_PORT}" \ --disaggregation-mode decode & # Register prefill separately so the frontend routes each disaggregated stage. DYN_SYSTEM_PORT="${DYN_SYSTEM_PORT2:-8082}" \ - dynamo-vllm-sidecar \ + python3 -m dynamo.vllm.sidecar \ --grpc-endpoint "127.0.0.1:${VLLM_PREFILL_GRPC_PORT}" \ --component prefill \ --disaggregation-mode prefill & diff --git a/tests/serve/test_vllm.py b/tests/serve/test_vllm.py index b9cc9c3a2b9c..a2cd13255a30 100644 --- a/tests/serve/test_vllm.py +++ b/tests/serve/test_vllm.py @@ -9,9 +9,11 @@ from dataclasses import dataclass, field import pytest +import requests from tests.serve.common import ( WORKSPACE_DIR, + managed_serve_deployment, params_with_model_mark, run_serve_deployment, ) @@ -21,8 +23,9 @@ VLLM_MULTIMODAL_PROFILES, VLLM_TOPOLOGY_SCRIPTS, ) -from tests.utils.constants import DefaultPort +from tests.utils.constants import DefaultPort, DynamoPortRange from tests.utils.engine_process import EngineConfig +from tests.utils.gpu_args import map_cuda_visible_devices from tests.utils.multimodal import make_multimodal_configs from tests.utils.payload_builder import ( chat_payload, @@ -45,6 +48,7 @@ EmbeddingPayload, ToolCallingChatPayload, ) +from tests.utils.port_utils import reserved_ports logger = logging.getLogger(__name__) @@ -857,6 +861,86 @@ def test_serve_deployment( run_serve_deployment(config, request, ports=dynamo_dynamic_ports) +@pytest.mark.vllm +@pytest.mark.e2e +@pytest.mark.gpu_1 +@pytest.mark.post_merge +@pytest.mark.core +@pytest.mark.profiled_vram_gib(8.0) # ~5.7 GiB observed; headroom for two engines. +@pytest.mark.model("Qwen/Qwen3-0.6B") +@pytest.mark.requested_vllm_kv_cache_bytes(1_119_388_000) +@pytest.mark.timeout(600) +@pytest.mark.parametrize("num_system_ports", [2], indirect=True) +def test_native_sidecar_prefill_decode_handoff( + request, + runtime_services_dynamic_ports, + dynamo_dynamic_ports, + predownload_models, +): + """Exercise NIXL metadata through the real vllm-rs gRPC receiver.""" + import vllm + + ports = dynamo_dynamic_ports + device = map_cuda_visible_devices([0], os.environ.get("CUDA_VISIBLE_DEVICES")) + with reserved_ports(4, DynamoPortRange.SERVE.value) as native_ports: + config = VLLMConfig( + name="native_sidecar_prefill_decode", + directory=os.path.join(WORKSPACE_DIR, "lib/sidecar/vllm"), + script_name="disagg.sh", + model="Qwen/Qwen3-0.6B", + marks=[], + request_payloads=[], + health_check_workers=True, + script_args=[ + "--kv-transfer-config", + ( + '{"kv_connector":"NixlConnector","kv_role":"kv_both",' + '"kv_load_failure_policy":"fail"}' + ), + ], + env={ + # Both engines fit on one GPU; the gRPC handoff is unchanged. + "DYNAMO_HOME": WORKSPACE_DIR, + "VLLM_DECODE_GPU": device, + "VLLM_PREFILL_GPU": device, + "VLLM_PLUGINS": "", + "VLLM_RUST_FRONTEND_PATH": os.path.join( + os.path.dirname(vllm.__file__), "vllm-rs" + ), + "DYN_HEALTH_CHECK_ENABLED": "true", + "VLLM_DECODE_HTTP_PORT": str(native_ports[0]), + "VLLM_DECODE_GRPC_PORT": str(native_ports[1]), + "VLLM_PREFILL_HTTP_PORT": str(native_ports[2]), + "VLLM_PREFILL_GRPC_PORT": str(native_ports[3]), + "VLLM_DECODE_NIXL_SIDE_CHANNEL_PORT": str( + ports.nixl_side_channel_ports[0] + ), + "VLLM_PREFILL_NIXL_SIDE_CHANNEL_PORT": str( + ports.nixl_side_channel_ports[1] + ), + "VLLM_PREFILL_KV_EVENT_PORT": str(ports.kv_event_port), + }, + ) + with managed_serve_deployment(config, request, ports=ports): + response = requests.post( + f"http://localhost:{ports.frontend_port}/v1/completions", + json={ + "model": config.model, + # Span several KV blocks so decode must consume remote KV. + "prompt": "The quick brown fox jumps over the lazy dog. " * 64, + "max_tokens": 8, + "temperature": 0, + "ignore_eos": True, + }, + timeout=60, + ) + assert response.status_code == 200, response.text + result = response.json() + assert result["usage"]["prompt_tokens"] > 256, result + assert result["usage"]["completion_tokens"] == 8, result + assert result["choices"][0]["finish_reason"] == "length", result + + # LoRA Test Directory lora_dir = os.path.join(vllm_dir, "launch/lora")