Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions vime/backends/vllm_utils/vllm_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,8 @@ def _wait_server_healthy(base_url: str, process: multiprocessing.Process | None)
if process is not None and not process.is_alive():
raise RuntimeError(f"vLLM server exited unexpectedly with code {process.exitcode}")
time.sleep(2)


class VLLMEngine(RayActor):
"""Ray actor for vLLM OpenAI HTTP rollout (connect or spawn local ``vllm serve``)."""

Expand Down
7 changes: 4 additions & 3 deletions vime/ray/rollout.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,17 @@
import os
import random
import time
from argparse import Namespace
from pathlib import Path
from typing import Any

import numpy as np
import ray
import torch
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy

from vime.backends.vllm_utils.vllm_config import ModelConfig, ServerGroupConfig, VllmConfig
from vime.backends.vllm_utils.vllm_engine import VLLMEngine

# Memory-type tag strings shared with the vLLM engine's sleep/wake_up API.
GPU_MEMORY_TYPE_KV_CACHE = "kv_cache"
GPU_MEMORY_TYPE_WEIGHTS = "weights"
Expand Down Expand Up @@ -919,10 +920,10 @@ def _start_router(args, *, has_pd_disaggregation: bool = False, force_new: bool
if router_port is None:
router_port = find_available_port(random.randint(3000, 4000))

from vime.utils.http_utils import run_router

from vllm_router.router_args import RouterArgs

from vime.utils.http_utils import run_router

router_args = RouterArgs.from_cli_args(args, use_router_prefix=True)

router_args.host = router_ip
Expand Down
Loading