diff --git a/.github/workflows/nightly-benchmark.yml b/.github/workflows/nightly-benchmark.yml index 19a4049c28..a0226603e3 100644 --- a/.github/workflows/nightly-benchmark.yml +++ b/.github/workflows/nightly-benchmark.yml @@ -3,6 +3,7 @@ name: Nightly Benchmark on: schedule: - cron: '0 8 * * *' + pull_request: workflow_dispatch: inputs: models: @@ -92,7 +93,7 @@ jobs: - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bSingle } variant: - { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false } - - { id: vllm, runtime: vllm, grpc_only: "true", setup_vllm: true, setup_trtllm: false } + - { id: vllm, runtime: vllm, grpc_only: "false", setup_vllm: true, setup_trtllm: false } steps: - name: Check filters @@ -199,7 +200,7 @@ jobs: - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bMulti } variant: - { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" } - - { id: vllm, runtime: vllm, grpc_only: "true", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" } + - { id: vllm, runtime: vllm, grpc_only: "false", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" } steps: - name: Check filters @@ -304,7 +305,7 @@ jobs: - { id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8, slug: meta-llama-Llama-4-Maverick-17B-128E-Instruct-FP8, test_class: TestNightlyLlama4MaverickSingle } variant: - { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" } - - { id: vllm, runtime: vllm, grpc_only: "true", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" } + - { id: vllm, runtime: vllm, grpc_only: "false", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" } steps: - name: Check filters diff --git a/e2e_test/benchmarks/test_nightly_perf.py b/e2e_test/benchmarks/test_nightly_perf.py index 7c0838814e..1879ab0460 100644 --- a/e2e_test/benchmarks/test_nightly_perf.py +++ b/e2e_test/benchmarks/test_nightly_perf.py @@ -6,7 +6,8 @@ Each model has Single (1 worker) and Multi (N workers) classes, both parametrized with http and grpc backends. The workflow matrix crosses -model × variant (single/multi × sglang/vllm), filtering vllm to grpc-only. +model × variant (single/multi × sglang/vllm); both protocols run for +all runtimes. genai-bench defaults (omitted flags): - Concurrency: [1, 2, 4, 8, 16, 32, 64, 128, 256] @@ -29,7 +30,7 @@ _MAX_TIME_PER_RUN = 10 # seconds per scenario×concurrency combo _TIMEOUT_SEC = 1440 * 60 # match workflow timeout-minutes -_TEST_MODE = False +_TEST_MODE = True # TODO: remove this when we want to run the full benchmark _TEST_NUM_CONCURRENCY = 1 _TEST_TRAFFIC_SCENARIO = "D(100,100)" _TEST_MAX_REQUESTS = 10 diff --git a/e2e_test/infra/model_pool.py b/e2e_test/infra/model_pool.py index fe04f756f1..b176d10556 100644 --- a/e2e_test/infra/model_pool.py +++ b/e2e_test/infra/model_pool.py @@ -38,6 +38,14 @@ logger = logging.getLogger(__name__) +# Shared vLLM defaults for both gRPC and HTTP modes. +_VLLM_DEFAULT_ARGS = [ + "--max-model-len", + "16384", + "--gpu-memory-utilization", + "0.9", +] + @dataclass(frozen=True) class WorkerIdentity: @@ -186,13 +194,18 @@ def _http_health_check(self, timeout: float = 5.0) -> bool: def deep_health_check(self, timeout: float = 30.0) -> bool: """Deep health check that verifies the model can actually generate. - Uses /health_generate for HTTP workers (runs actual inference). + Uses /health_generate for SGLang HTTP workers (runs actual inference). + For vLLM HTTP workers, falls back to /health (no /health_generate). For gRPC workers, falls back to standard health check. """ if self.mode == ConnectionMode.GRPC: # For gRPC, use standard health check (no /health_generate equivalent) return self._grpc_health_check(timeout) + # vLLM HTTP does not support /health_generate, use /health instead + if self.mode == ConnectionMode.HTTP and is_vllm(): + return self._http_health_check(timeout) + try: resp = httpx.get(f"{self.base_url}/health_generate", timeout=timeout) return resp.status_code == 200 @@ -517,6 +530,24 @@ def _launch_model( assert instance is not None return instance + # vLLM HTTP: use dedicated vLLM OpenAI-compatible API server + if mode == ConnectionMode.HTTP and is_vllm(): + logger.info( + "HTTP worker requested for %s: routing to vLLM HTTP backend", + model_id, + ) + spec = get_model_spec(model_id) + assert gpu_slot is not None + instance = self._launch_vllm_http_worker( + model_id=model_id, + model_spec=spec, + gpu_slot=gpu_slot, + startup_timeout=600, + instance_key=instance_key, + ) + assert instance is not None + return instance + spec = get_model_spec(model_id) model_path = spec["model"] tp_size = spec.get("tp", 1) @@ -1166,11 +1197,8 @@ def _build_grpc_cmd( str(port), "--tensor-parallel-size", str(tp_size), - "--max-model-len", - "16384", - "--gpu-memory-utilization", - "0.9", ] + cmd.extend(_VLLM_DEFAULT_ARGS) extra = model_spec.get("vllm_args", []) elif runtime == "trtllm": cmd = [ @@ -1197,6 +1225,45 @@ def _build_grpc_cmd( cmd.extend(extra) return cmd + @staticmethod + def _build_vllm_http_cmd( + model_path: str, + host: str, + port: int, + tp_size: int, + model_spec: dict, + ) -> list[str]: + """Build the vLLM HTTP (OpenAI-compatible) server launch command. + + Args: + model_path: HuggingFace model path. + host: Host to bind to. + port: Port to bind to. + tp_size: Tensor parallel size. + model_spec: Model specification dict (for vllm_args). + + Returns: + Command list for subprocess.Popen. + """ + cmd = [ + "python3", + "-m", + "vllm.entrypoints.openai.api_server", + "--model", + model_path, + "--host", + host, + "--port", + str(port), + "--tensor-parallel-size", + str(tp_size), + ] + cmd.extend(_VLLM_DEFAULT_ARGS) + extra = model_spec.get("vllm_args", []) + if extra: + cmd.extend(extra) + return cmd + def _launch_grpc_worker( self, runtime: str, @@ -1392,6 +1459,99 @@ def get_grpc_worker( instance.acquire() return instance + def _launch_vllm_http_worker( + self, + model_id: str, + model_spec: dict, + gpu_slot: GPUSlot, + startup_timeout: int, + instance_key: str | None = None, + ) -> ModelInstance | None: + """Launch a vLLM HTTP (OpenAI-compatible) worker. + + Args: + model_id: Model identifier. + model_spec: Model specification dict from MODEL_SPECS. + gpu_slot: GPU slot assignment. + startup_timeout: Timeout for worker to become healthy. + instance_key: Custom instance key, or None to auto-generate. + + Returns: + The launched ModelInstance, or None if launch fails. + """ + model_path = model_spec["model"] + tp_size = model_spec.get("tp", 1) + port = gpu_slot.port + assert port is not None + + env = os.environ.copy() + env["CUDA_VISIBLE_DEVICES"] = gpu_slot.cuda_visible_devices() + + cmd = self._build_vllm_http_cmd(model_path, DEFAULT_HOST, port, tp_size, model_spec) + + key = instance_key or f"{model_id}:http" + logger.info( + "Launching vLLM HTTP worker %s on GPUs %s port %d", + key, + gpu_slot.gpu_ids, + port, + ) + + show_output = os.environ.get(ENV_SHOW_WORKER_LOGS, "0") == "1" + + # Determine output targets: terminal, log file, or devnull + stdout_target: int | IO[Any] | None = None + stderr_target: int | IO[Any] | None = None + if not show_output: + if self.log_dir: + os.makedirs(self.log_dir, exist_ok=True) + safe_key = key.replace("/", "__").replace(":", "_") + log_file = open(os.path.join(self.log_dir, f"worker-{safe_key}-{port}.log"), "w") + self._log_files[key] = log_file + stdout_target = log_file + stderr_target = subprocess.STDOUT + else: + stdout_target = subprocess.DEVNULL + stderr_target = subprocess.DEVNULL + + try: + proc = subprocess.Popen( + cmd, + env=env, + stdout=stdout_target, + stderr=stderr_target, + start_new_session=True, + ) + except Exception: + lf = self._log_files.pop(key, None) + if lf is not None: + lf.close() + raise + + base_url = f"http://{DEFAULT_HOST}:{port}" + instance = ModelInstance( + model_id=model_id, + mode=ConnectionMode.HTTP, + model_path=model_path, + base_url=base_url, + port=port, + process=proc, + gpu_slot=gpu_slot, + key=key, + worker_type=WorkerType.REGULAR, + bootstrap_port=None, + last_used=time.time(), + ) + self.instances[key] = instance + + try: + self._wait_worker_healthy(instance, startup_timeout) + return instance + except Exception as e: + logger.error("Failed to start vLLM HTTP worker %s: %s", key, e) + self._evict_instance(key) + return None + def launch_workers( self, workers: list[WorkerIdentity],