feat(benchmark): add vLLM HTTP to nightly benchmark

.github/workflows/nightly-benchmark.yml

@@ -3,6 +3,7 @@ name: Nightly Benchmark
 on:
   schedule:
     - cron: '0 8 * * *'
+  pull_request:
   workflow_dispatch:
     inputs:
       models:
@@ -92,7 +93,7 @@ jobs:
           - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bSingle }
         variant:
           - { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false }
-          - { id: vllm,   runtime: vllm,   grpc_only: "true", setup_vllm: true, setup_trtllm: false }
+          - { id: vllm,   runtime: vllm,   grpc_only: "false", setup_vllm: true, setup_trtllm: false }
 
     steps:
       - name: Check filters
@@ -199,7 +200,7 @@ jobs:
           - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bMulti }
         variant:
           - { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" }
-          - { id: vllm,   runtime: vllm,   grpc_only: "true", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" }
+          - { id: vllm,   runtime: vllm,   grpc_only: "false", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" }
 
     steps:
       - name: Check filters
@@ -304,7 +305,7 @@ jobs:
           - { id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8, slug: meta-llama-Llama-4-Maverick-17B-128E-Instruct-FP8, test_class: TestNightlyLlama4MaverickSingle }
         variant:
           - { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" }
-          - { id: vllm,   runtime: vllm,   grpc_only: "true", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" }
+          - { id: vllm,   runtime: vllm,   grpc_only: "false", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" }
 
     steps:
       - name: Check filters

e2e_test/benchmarks/test_nightly_perf.py

@@ -6,7 +6,8 @@
 
 Each model has Single (1 worker) and Multi (N workers) classes, both
 parametrized with http and grpc backends. The workflow matrix crosses
-model × variant (single/multi × sglang/vllm), filtering vllm to grpc-only.
+model × variant (single/multi × sglang/vllm); both protocols run for
+all runtimes.
 
 genai-bench defaults (omitted flags):
   - Concurrency: [1, 2, 4, 8, 16, 32, 64, 128, 256]
@@ -29,7 +30,7 @@
 _MAX_TIME_PER_RUN = 10  # seconds per scenario×concurrency combo
 _TIMEOUT_SEC = 1440 * 60  # match workflow timeout-minutes
 
-_TEST_MODE = False
+_TEST_MODE = True # TODO: remove this when we want to run the full benchmark
 _TEST_NUM_CONCURRENCY = 1
 _TEST_TRAFFIC_SCENARIO = "D(100,100)"
 _TEST_MAX_REQUESTS = 10

e2e_test/infra/model_pool.py

@@ -38,6 +38,14 @@
 
 logger = logging.getLogger(__name__)
 
+# Shared vLLM defaults for both gRPC and HTTP modes.
+_VLLM_DEFAULT_ARGS = [
+    "--max-model-len",
+    "16384",
+    "--gpu-memory-utilization",
+    "0.9",
+]
+
 
 @dataclass(frozen=True)
 class WorkerIdentity:
@@ -186,13 +194,18 @@ def _http_health_check(self, timeout: float = 5.0) -> bool:
     def deep_health_check(self, timeout: float = 30.0) -> bool:
         """Deep health check that verifies the model can actually generate.
 
-        Uses /health_generate for HTTP workers (runs actual inference).
+        Uses /health_generate for SGLang HTTP workers (runs actual inference).
+        For vLLM HTTP workers, falls back to /health (no /health_generate).
         For gRPC workers, falls back to standard health check.
         """
         if self.mode == ConnectionMode.GRPC:
             # For gRPC, use standard health check (no /health_generate equivalent)
             return self._grpc_health_check(timeout)
 
+        # vLLM HTTP does not support /health_generate, use /health instead
+        if self.mode == ConnectionMode.HTTP and is_vllm():
+            return self._http_health_check(timeout)
+
         try:
             resp = httpx.get(f"{self.base_url}/health_generate", timeout=timeout)
             return resp.status_code == 200
@@ -517,6 +530,24 @@ def _launch_model(
                 assert instance is not None
                 return instance
 
+        # vLLM HTTP: use dedicated vLLM OpenAI-compatible API server
+        if mode == ConnectionMode.HTTP and is_vllm():
+            logger.info(
+                "HTTP worker requested for %s: routing to vLLM HTTP backend",
+                model_id,
+            )
+            spec = get_model_spec(model_id)
+            assert gpu_slot is not None
+            instance = self._launch_vllm_http_worker(
+                model_id=model_id,
+                model_spec=spec,
+                gpu_slot=gpu_slot,
+                startup_timeout=600,
+                instance_key=instance_key,
+            )
+            assert instance is not None
+            return instance
+
         spec = get_model_spec(model_id)
         model_path = spec["model"]
         tp_size = spec.get("tp", 1)
@@ -1166,11 +1197,8 @@ def _build_grpc_cmd(
                 str(port),
                 "--tensor-parallel-size",
                 str(tp_size),
-                "--max-model-len",
-                "16384",
-                "--gpu-memory-utilization",
-                "0.9",
             ]
+            cmd.extend(_VLLM_DEFAULT_ARGS)
             extra = model_spec.get("vllm_args", [])
         elif runtime == "trtllm":
             cmd = [
@@ -1197,6 +1225,45 @@ def _build_grpc_cmd(
             cmd.extend(extra)
         return cmd
 
+    @staticmethod
+    def _build_vllm_http_cmd(
+        model_path: str,
+        host: str,
+        port: int,
+        tp_size: int,
+        model_spec: dict,
+    ) -> list[str]:
+        """Build the vLLM HTTP (OpenAI-compatible) server launch command.
+
+        Args:
+            model_path: HuggingFace model path.
+            host: Host to bind to.
+            port: Port to bind to.
+            tp_size: Tensor parallel size.
+            model_spec: Model specification dict (for vllm_args).
+
+        Returns:
+            Command list for subprocess.Popen.
+        """
+        cmd = [
+            "python3",
+            "-m",
+            "vllm.entrypoints.openai.api_server",
+            "--model",
+            model_path,
+            "--host",
+            host,
+            "--port",
+            str(port),
+            "--tensor-parallel-size",
+            str(tp_size),
+        ]
+        cmd.extend(_VLLM_DEFAULT_ARGS)
+        extra = model_spec.get("vllm_args", [])
+        if extra:
+            cmd.extend(extra)
+        return cmd
+
     def _launch_grpc_worker(
         self,
         runtime: str,
@@ -1392,6 +1459,99 @@ def get_grpc_worker(
             instance.acquire()
             return instance
 
+    def _launch_vllm_http_worker(
+        self,
+        model_id: str,
+        model_spec: dict,
+        gpu_slot: GPUSlot,
+        startup_timeout: int,
+        instance_key: str | None = None,
+    ) -> ModelInstance | None:
+        """Launch a vLLM HTTP (OpenAI-compatible) worker.
+
+        Args:
+            model_id: Model identifier.
+            model_spec: Model specification dict from MODEL_SPECS.
+            gpu_slot: GPU slot assignment.
+            startup_timeout: Timeout for worker to become healthy.
+            instance_key: Custom instance key, or None to auto-generate.
+
+        Returns:
+            The launched ModelInstance, or None if launch fails.
+        """
+        model_path = model_spec["model"]
+        tp_size = model_spec.get("tp", 1)
+        port = gpu_slot.port
+        assert port is not None
+
+        env = os.environ.copy()
+        env["CUDA_VISIBLE_DEVICES"] = gpu_slot.cuda_visible_devices()
+
+        cmd = self._build_vllm_http_cmd(model_path, DEFAULT_HOST, port, tp_size, model_spec)
+
+        key = instance_key or f"{model_id}:http"
+        logger.info(
+            "Launching vLLM HTTP worker %s on GPUs %s port %d",
+            key,
+            gpu_slot.gpu_ids,
+            port,
+        )
+
+        show_output = os.environ.get(ENV_SHOW_WORKER_LOGS, "0") == "1"
+
+        # Determine output targets: terminal, log file, or devnull
+        stdout_target: int | IO[Any] | None = None
+        stderr_target: int | IO[Any] | None = None
+        if not show_output:
+            if self.log_dir:
+                os.makedirs(self.log_dir, exist_ok=True)
+                safe_key = key.replace("/", "__").replace(":", "_")
+                log_file = open(os.path.join(self.log_dir, f"worker-{safe_key}-{port}.log"), "w")
+                self._log_files[key] = log_file
+                stdout_target = log_file
+                stderr_target = subprocess.STDOUT
+            else:
+                stdout_target = subprocess.DEVNULL
+                stderr_target = subprocess.DEVNULL
+
+        try:
+            proc = subprocess.Popen(
+                cmd,
+                env=env,
+                stdout=stdout_target,
+                stderr=stderr_target,
+                start_new_session=True,
+            )
+        except Exception:
+            lf = self._log_files.pop(key, None)
+            if lf is not None:
+                lf.close()
+            raise
+
+        base_url = f"http://{DEFAULT_HOST}:{port}"
+        instance = ModelInstance(
+            model_id=model_id,
+            mode=ConnectionMode.HTTP,
+            model_path=model_path,
+            base_url=base_url,
+            port=port,
+            process=proc,
+            gpu_slot=gpu_slot,
+            key=key,
+            worker_type=WorkerType.REGULAR,
+            bootstrap_port=None,
+            last_used=time.time(),
+        )
+        self.instances[key] = instance
+
+        try:
+            self._wait_worker_healthy(instance, startup_timeout)
+            return instance
+        except Exception as e:
+            logger.error("Failed to start vLLM HTTP worker %s: %s", key, e)
+            self._evict_instance(key)
+            return None
+
     def launch_workers(
         self,
         workers: list[WorkerIdentity],