diff --git a/vime/backends/vllm_utils/vllm_engine.py b/vime/backends/vllm_utils/vllm_engine.py index f6fe89075..6458716f4 100644 --- a/vime/backends/vllm_utils/vllm_engine.py +++ b/vime/backends/vllm_utils/vllm_engine.py @@ -521,6 +521,8 @@ def _wait_server_healthy(base_url: str, process: multiprocessing.Process | None) if process is not None and not process.is_alive(): raise RuntimeError(f"vLLM server exited unexpectedly with code {process.exitcode}") time.sleep(2) + + class VLLMEngine(RayActor): """Ray actor for vLLM OpenAI HTTP rollout (connect or spawn local ``vllm serve``).""" diff --git a/vime/ray/rollout.py b/vime/ray/rollout.py index 69d6a42c9..ef5fb31b8 100644 --- a/vime/ray/rollout.py +++ b/vime/ray/rollout.py @@ -5,7 +5,6 @@ import os import random import time -from argparse import Namespace from pathlib import Path from typing import Any @@ -13,8 +12,10 @@ import ray import torch from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy + from vime.backends.vllm_utils.vllm_config import ModelConfig, ServerGroupConfig, VllmConfig from vime.backends.vllm_utils.vllm_engine import VLLMEngine + # Memory-type tag strings shared with the vLLM engine's sleep/wake_up API. GPU_MEMORY_TYPE_KV_CACHE = "kv_cache" GPU_MEMORY_TYPE_WEIGHTS = "weights" @@ -919,10 +920,10 @@ def _start_router(args, *, has_pd_disaggregation: bool = False, force_new: bool if router_port is None: router_port = find_available_port(random.randint(3000, 4000)) - from vime.utils.http_utils import run_router - from vllm_router.router_args import RouterArgs + from vime.utils.http_utils import run_router + router_args = RouterArgs.from_cli_args(args, use_router_prefix=True) router_args.host = router_ip