ci(vllm): upgrade CI to vllm>=0.22.1 (cu13 stack, flashinfer jit-cache, Qwen3 embedding)

e2e_test/embeddings/test_basic.py

@@ -20,7 +20,7 @@
 
 @pytest.mark.engine("sglang", "vllm")
 @pytest.mark.gpu(1)
-@pytest.mark.model("intfloat/e5-mistral-7b-instruct")
+@pytest.mark.model("Qwen/Qwen3-Embedding-0.6B")
 @pytest.mark.e2e
 @pytest.mark.parametrize("setup_backend", ["grpc", "http"], indirect=True)
 @pytest.mark.parametrize("api_client", ["openai", "smg"], indirect=True)

e2e_test/embeddings/test_correctness.py

@@ -128,7 +128,7 @@ def get_hf_st_embeddings(texts: str | list[str], model_path: str) -> np.ndarray:
     """Get embeddings using sentence-transformers library.
 
     This handles the correct pooling strategy for each model automatically.
-    For e5-mistral, it uses last-token pooling (not mean pooling).
+    For Qwen3-Embedding, it uses last-token pooling (not mean pooling).
 
     Uses CPU to compute reference embeddings to avoid GPU memory conflicts
     with the worker being tested.
@@ -174,7 +174,7 @@ def hf_reference_embeddings(request):
     from infra.model_specs import MODEL_SPECS
 
     # Get model path from MODEL_SPECS for the embedding model
-    model_path = MODEL_SPECS.get("intfloat/e5-mistral-7b-instruct", {}).get("model")
+    model_path = MODEL_SPECS.get("Qwen/Qwen3-Embedding-0.6B", {}).get("model")
     if model_path is None:
         pytest.skip("Embedding model not found in MODEL_SPECS")
 
@@ -211,7 +211,7 @@ def hf_reference_embeddings(request):
 @pytest.mark.engine("sglang", "vllm")
 @pytest.mark.skip_for_runtime("sglang", reason="sglang embedding output diverges from HF reference")
 @pytest.mark.gpu(1)
-@pytest.mark.model("intfloat/e5-mistral-7b-instruct")
+@pytest.mark.model("Qwen/Qwen3-Embedding-0.6B")
 @pytest.mark.e2e
 @pytest.mark.parametrize("setup_backend", ["grpc", "http"], indirect=True)
 class TestEmbeddingCorrectness:
@@ -263,9 +263,11 @@ def test_relevance_scores(self, setup_backend, api_client, hf_reference_embeddin
         and HuggingFace implementations within tolerance.
         """
         backend, model_path, _, _ = setup_backend
-        tolerance = 0.05
+        # 0.5 on the x100 scale = cosine 5e-3; covers cross-implementation noise
+        # (vLLM-vs-ST, bf16) while staying 2x tighter than upstream's per-vector 1e-2
+        tolerance = 0.5
 
-        # Format query with instruction (for e5-mistral)
+        # Format query with instruction (Qwen3-Embedding convention; docs stay unprefixed)
         query = (
             f"Instruct: Given a search query, retrieve relevant passages that answer the query\n"
             f"Query: {RELEVANCE_TEST_DATA['sample_query']}"

e2e_test/infra/model_specs.py

@@ -91,9 +91,9 @@ def _resolve_model_path(hf_path: str) -> str:
             '{"disable_any_whitespace": true, "backend": "xgrammar"}',
         ],
     },
-    # Embedding model
-    "intfloat/e5-mistral-7b-instruct": {
-        "model": _resolve_model_path("intfloat/e5-mistral-7b-instruct"),
+    # Embedding model (last-token pooling; upstream-tested under transformers v5)
+    "Qwen/Qwen3-Embedding-0.6B": {
+        "model": _resolve_model_path("Qwen/Qwen3-Embedding-0.6B"),
         "tp": 1,
         "features": ["embedding"],
     },
@@ -261,7 +261,7 @@ def get_model_spec(model_id: str) -> dict:
     "model"
 ]
 DEFAULT_GPT_OSS_MODEL_PATH = MODEL_SPECS["openai/gpt-oss-20b"]["model"]
-DEFAULT_EMBEDDING_MODEL_PATH = MODEL_SPECS["intfloat/e5-mistral-7b-instruct"]["model"]
+DEFAULT_EMBEDDING_MODEL_PATH = MODEL_SPECS["Qwen/Qwen3-Embedding-0.6B"]["model"]
 
 
 # =============================================================================

scripts/ci_install_vllm.sh

@@ -19,28 +19,39 @@ fi
 
 echo "Using uv version: $(uv --version)"
 
-# Pin vLLM below 0.19.1. vLLM 0.19.1 bundled the transformers v5 upgrade,
-# which breaks intfloat/e5-mistral-7b-instruct embedding quality
-# (self-similarity ~0.33 instead of ~1.0) via an EOS / last-token pooling
-# regression. Last-known-good combo is:
-#   vllm==0.19.0 + transformers==4.57.6   (run 24591985132, 82a3fb1a)
-# Regression first appeared in run 24608587304 (dcede344). Failure signature:
-# https://github.com/lightseekorg/smg/actions/runs/24644816475/job/72068881582
-#
-# We rely on vllm 0.19.0's own wheel metadata (transformers<5,>=4.56.0) to
-# force transformers back to 4.x. Not pinning transformers separately — if
-# we ship vllm we don't want to second-guess the library's own dep range;
-# drop this pin whenever vllm publishes a release that re-verifies embedding
-# correctness post-transformers-v5.
+# Floor 0.22.1: older vllm resolved an early transformers v5 that broke
+# e5-mistral last-token pooling (the old <0.19.1 pin); 0.22.1+ only admits
+# transformers >= 5.5.1. e2e-1gpu-embeddings is the quality gate.
+# --torch-backend=auto matches the torch CUDA variant to the pod's driver.
 echo "Installing vLLM..."
-uv pip install "vllm<0.19.1"
+uv pip install "vllm>=0.22.1" --torch-backend=auto
 
-# Install nixl for vLLM PD disaggregation (NIXL KV transfer).
-# Pin <1.1.0: 1.1.0 lands a cu13 nixl_ep_cpp.so that vLLM 0.19.0 imports
-# unconditionally, breaking CUDA-12 ARC pods. Drop once vLLM or the
-# runner pool catches up.
+# NIXL for vLLM PD disaggregation. The bare metapackage pulls both cu12 and
+# cu13 backends, so install the top-level shim alone, then the backend
+# matching torch's CUDA (same normalization as vLLM's own CI).
 echo "Installing nixl..."
-uv pip install "nixl<1.1.0"
+CUDA_MAJOR=$(python3 -c "import torch; print(torch.version.cuda.split('.')[0])")
+uv pip install --no-deps "nixl>=1.2.0"
+uv pip install "nixl-cu${CUDA_MAJOR}>=1.2.0"
+
+# Remove nixl_ep (MoE all-to-all, unused in CI): vLLM imports it eagerly when
+# present, tying every worker startup to its extra native deps
+SITE_PACKAGES=$(python3 -c "import sysconfig; print(sysconfig.get_paths()['platlib'])")
+rm -rf "${SITE_PACKAGES}/nixl_ep"
+
+# Import canary: fail here (not mid-e2e) if the nixl install is broken
+# (torch first so its bundled CUDA libraries are loaded)
+python3 -c "import torch, nixl"
+echo "nixl import canary OK"
+
+# FlashInfer JIT cache: vLLM JIT-compiles flashinfer kernels at engine startup
+# and the pods have no CUDA toolchain — install the precompiled cache instead,
+# same recipe as vLLM's own Dockerfile.
+echo "Installing flashinfer-jit-cache..."
+CUDA_TAG=$(python3 -c "import torch; print(torch.version.cuda.replace('.', ''))")
+FLASHINFER_VERSION=$(python3 -c "import importlib.metadata as m; print(m.version('flashinfer-python'))")
+uv pip install "flashinfer-jit-cache==${FLASHINFER_VERSION}" \
+    --index-url "https://flashinfer.ai/whl/cu${CUDA_TAG}"
 
 # Install gRPC packages from source (not PyPI) so PR changes are always tested
 echo "Installing smg-grpc-proto and smg-grpc-servicer from source..."