From 2c4402526c5c327905433b4b5990f520dd44bdb4 Mon Sep 17 00:00:00 2001 From: "jiang1.li" Date: Fri, 31 Jul 2026 13:25:28 +0000 Subject: [PATCH 1/5] [CPU] Make SGL AMX kernels unconditional, drop VLLM_CPU_SGL_KERNEL flag The SGLang-derived AMX kernels for INT8 W8A8 linear (symmetric, shape-aligned) and unquantized MoE were gated behind an experimental opt-in flag. Benchmarking shows no downside to enabling them unconditionally in their existing eligibility window, so drop the flag and always use them when eligible. oneDNN remains the fallback for all ineligible cases (asymmetric quant, misaligned shapes, non-AMX x86) and is untouched for unquantized FP32/FP16/BF16 GEMM, which continues to use oneDNN exclusively as before. Tested: 279 tests pass (test_onednn.py, CPU unquantized GEMM dispatch, scaled_mm kernel selection, CPU FP8 scaled_mm, W8A8 e2e model test). Offline-inference smoke test across Llama-3.1-8B-Instruct, gemma-7b, Qwen3-8B, and RedHatAI/Meta-Llama-3.1-8B-quantized.w8a8 all produce coherent output. Benchmarked the W8A8 model (1024/1024 in/out, 15 prompts) with the flag removed vs. oneDNN baseline (flag off): ~10% lower mean TTFT (4136.7ms -> 3721.5ms), ~6% lower mean TPOT (79.15ms -> 74.09ms), ~7% higher total throughput (359.8 -> 385.1 tok/s). Signed-off-by: jiang1.li --- docs/getting_started/installation/cpu.md | 1 - vllm/envs.py | 3 --- .../kernels/linear/scaled_mm/cpu.py | 2 -- vllm/model_executor/layers/utils.py | 20 +------------------ 4 files changed, 1 insertion(+), 25 deletions(-) diff --git a/docs/getting_started/installation/cpu.md b/docs/getting_started/installation/cpu.md index 3e6227b8d403..ce7562478f72 100644 --- a/docs/getting_started/installation/cpu.md +++ b/docs/getting_started/installation/cpu.md @@ -152,7 +152,6 @@ VLLM_USE_PRECOMPILED=1 VLLM_PRECOMPILED_WHEEL_VARIANT=cpu VLLM_TARGET_DEVICE=cpu - `VLLM_CPU_OMP_THREADS_BIND`: specify the CPU cores dedicated to the OpenMP threads, can be set as CPU id lists, `auto` (by default), or `nobind` (to disable binding to individual CPU cores and to inherit user-defined OpenMP variables). For example, `VLLM_CPU_OMP_THREADS_BIND=0-31` means there will be 32 OpenMP threads bound on 0-31 CPU cores. `VLLM_CPU_OMP_THREADS_BIND=0-31|32-63` means there will be 2 tensor parallel processes, 32 OpenMP threads of rank0 are bound on 0-31 CPU cores, and the OpenMP threads of rank1 are bound on 32-63 CPU cores. By setting to `auto`, the OpenMP threads of each rank are bound to the CPU cores in each NUMA node respectively. If set to `nobind`, the number of OpenMP threads is determined by the standard `OMP_NUM_THREADS` environment variable. - `VLLM_CPU_NUM_OF_RESERVED_CPU`: specify the number of CPU cores which are not dedicated to the OpenMP threads for each rank. The variable only takes effect when VLLM_CPU_OMP_THREADS_BIND is set to `auto`. Default value is `None`. If the value is not set and use `auto` thread binding, no CPU will be reserved for `world_size == 1`, 1 CPU per rank will be reserved for `world_size > 1`. - `CPU_VISIBLE_MEMORY_NODES`: specify visible NUMA memory nodes for vLLM CPU workers, similar to ```CUDA_VISIBLE_DEVICES```. The variable only takes effect when VLLM_CPU_OMP_THREADS_BIND is set to `auto`. The variable provides more control for the auto thread-binding feature, such as masking nodes and changing nodes binding sequence. -- `VLLM_CPU_SGL_KERNEL` (x86 only, Experimental): whether to use small-batch optimized kernels for the linear layer, especially for low-latency requirements like online serving. The kernels require AMX instruction set, BFloat16 weight type and weight shapes divisible by 32. Default is `0` (False). MoE layers always use the grouped-gemm kernels on x86 and are unaffected by this variable. - `VLLM_ZENTORCH_WEIGHT_PREPACK` (AMD Zen only): when `ZenCpuPlatform` is active, eagerly prepack linear weights into ZenDNN's blocked layout at model load time, eliminating per-inference layout conversion overhead. Default is `1` (enabled). See [AMD Zen optimizations](#amd-zen-optimizations). ## FAQ diff --git a/vllm/envs.py b/vllm/envs.py index 792a2084b36d..c77184161baa 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -51,7 +51,6 @@ VLLM_CPU_KVCACHE_SPACE: int | None = 0 VLLM_CPU_OMP_THREADS_BIND: str = "auto" VLLM_CPU_NUM_OF_RESERVED_CPU: int | None = None - VLLM_CPU_SGL_KERNEL: bool = False VLLM_CPU_ATTN_SPLIT_KV: bool = True VLLM_ZENTORCH_WEIGHT_PREPACK: bool = True VLLM_CPU_INT4_W4A8: bool = True @@ -868,8 +867,6 @@ def _resolve_rust_cli_path() -> str | None: if "VLLM_CPU_NUM_OF_RESERVED_CPU" in os.environ else None ), - # (CPU backend only) whether to use SGL kernels, optimized for small batch. - "VLLM_CPU_SGL_KERNEL": lambda: bool(int(os.getenv("VLLM_CPU_SGL_KERNEL", "0"))), # (CPU backend only) whether to enable attention spilt KV. "VLLM_CPU_ATTN_SPLIT_KV": lambda: bool( int(os.getenv("VLLM_CPU_ATTN_SPLIT_KV", "1")) diff --git a/vllm/model_executor/kernels/linear/scaled_mm/cpu.py b/vllm/model_executor/kernels/linear/scaled_mm/cpu.py index 083cb473aaca..cb0fe0325b6d 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/cpu.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/cpu.py @@ -5,7 +5,6 @@ import torch from vllm import _custom_ops as ops -from vllm import envs from vllm.model_executor.layers.quantization.utils import replace_parameter from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( convert_to_channelwise, @@ -44,7 +43,6 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: N, K = weight.size() if ( current_platform.get_cpu_architecture() == CpuArchEnum.X86 - and envs.VLLM_CPU_SGL_KERNEL and self.config.input_symmetric and check_cpu_sgl_kernel(N, K, dtype) ): diff --git a/vllm/model_executor/layers/utils.py b/vllm/model_executor/layers/utils.py index f7e0b6510547..227f617f5dbe 100644 --- a/vllm/model_executor/layers/utils.py +++ b/vllm/model_executor/layers/utils.py @@ -263,9 +263,6 @@ def dispatch_cpu_unquantized_gemm( layer.weight.data = ops.causal_conv1d_weight_pack(unpacked) return - N, K = layer.weight.size() - dtype = layer.weight.dtype - # Zen CPU path: zentorch_linear_unary with optional eager weight prepacking. if current_platform.is_zen_cpu() and hasattr( torch.ops.zentorch, "zentorch_linear_unary" @@ -294,22 +291,7 @@ def dispatch_cpu_unquantized_gemm( ) return - if envs.VLLM_CPU_SGL_KERNEL and check_cpu_sgl_kernel(N, K, dtype): - packed_weight = torch.ops._C.convert_weight_packed(layer.weight) - if getattr(layer, "bias", None) is not None: - bias_f32 = layer.bias.to(torch.float32) - else: - bias_f32 = None - layer.cpu_linear = lambda x, weight, bias: torch.ops._C.weight_packed_linear( - x, packed_weight, bias_f32 if bias is not None else None, True - ) - if remove_weight: - layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False) - logger.debug_once( - "CPU unquantized GEMM dispatch: using sgl-kernel weight_packed_linear" - ) - return - elif ( + if ( ops._supports_onednn and current_platform.get_cpu_architecture() != CpuArchEnum.POWERPC ): From f09e979d66f9d0c75efe30251ff2325a87076d04 Mon Sep 17 00:00:00 2001 From: "jiang1.li" Date: Mon, 3 Aug 2026 03:35:07 +0000 Subject: [PATCH 2/5] [CPU] Bump vendored oneDNN to v3.13 Move the x86 FetchContent pin from v3.10 to v3.13 to pick up upstream GEMM/threading fixes and improvements. ARM keeps its separately pinned commit, unchanged. Tested: 279 tests pass (test_onednn.py, CPU unquantized GEMM dispatch, scaled_mm kernel selection, CPU FP8 scaled_mm, W8A8 e2e model test). Offline-inference smoke test across Llama-3.1-8B-Instruct, gemma-7b, and Qwen3-8B is coherent on both versions (Llama and Qwen3 produce byte-identical greedy output vs. v3.10; gemma-7b diverges on one of four canned prompts, expected BF16 rounding-order sensitivity, still coherent). Benchmarked serving (1024/1024 in/out, 6 prompts) v3.10 vs. v3.13: TTFT (mean) TPOT (mean) total tok/s Llama-3.1-8B 2750->2898 ms 103.3->100.1 ms 113.2->116.6 gemma-7b 3255->3092 ms 133.3->130.4 ms 87.9->89.9 Qwen3-8B 2812->2806 ms 106.3->102.4 ms 110.0->114.1 No regression; TPOT and throughput improve slightly (~2-3%) across all three models, TTFT is within run-to-run noise for a 6-prompt sample. Signed-off-by: jiang1.li --- cmake/cpu_extension.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake index c66854386c4b..581d6003f504 100644 --- a/cmake/cpu_extension.cmake +++ b/cmake/cpu_extension.cmake @@ -345,7 +345,7 @@ if (ENABLE_X86_ISA OR (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) OR POWER9_FOUND FetchContent_Declare( oneDNN GIT_REPOSITORY https://github.com/oneapi-src/oneDNN.git - GIT_TAG v3.10 + GIT_TAG v3.13 GIT_PROGRESS TRUE GIT_SHALLOW TRUE ) From c00634da95ce40e992867e2162717d54ee51dca3 Mon Sep 17 00:00:00 2001 From: "jiang1.li" Date: Mon, 3 Aug 2026 03:58:48 +0000 Subject: [PATCH 3/5] [CPU] Add float16 to check_cpu_sgl_kernel eligibility gate The gate excluded torch.float16, but the underlying kernels (weight_packed_linear, int8_scaled_mm_with_quant, fused_experts_cpu in csrc/cpu/sgl-kernels/) all dispatch via AT_DISPATCH_REDUCED_FLOATING_TYPES, which covers both BFloat16 and Half with no additional dtype restriction in any of their TORCH_CHECK guards. Confirmed against the actual vendored kernel source (byte-identical to sglang's latest upstream main as of this commit) rather than assumed. Concretely this means FP16 weights/activations were being routed to the slower fallback path even on AMX hardware where the SGL kernel already handles them correctly. Signed-off-by: jiang1.li --- vllm/model_executor/layers/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/model_executor/layers/utils.py b/vllm/model_executor/layers/utils.py index 227f617f5dbe..1f65a21a4f39 100644 --- a/vllm/model_executor/layers/utils.py +++ b/vllm/model_executor/layers/utils.py @@ -225,7 +225,7 @@ def rocm_unquantized_gemm( def check_cpu_sgl_kernel(n: int, k: int, dtype: torch.dtype) -> bool: return ( torch.cpu._is_amx_tile_supported() - and (dtype in (torch.bfloat16, torch.int8)) + and (dtype in (torch.bfloat16, torch.float16, torch.int8)) and k % 32 == 0 and n % 16 == 0 ) From 8ce419a52c06c57ea06a38246afdffadb0814a75 Mon Sep 17 00:00:00 2001 From: "jiang1.li" Date: Mon, 3 Aug 2026 04:13:28 +0000 Subject: [PATCH 4/5] [CPU] Require AMX support in quantized CPU MoE experts' device check CPUExpertsFp8, CPUExpertsMxfp4, and CPUExpertsInt4's _supports_current_device() only checked current_platform.is_cpu(), with no x86 or AMX check; CPUExpertsInt8 checked x86 but not AMX either. All four unconditionally call fused_experts_cpu / torch.ops._C.convert_weight_packed in apply(), which per cmake/cpu_extension.cmake's VLLM_EXT_SRC_SGL are compiled only into the AMX-tier _C extension, not _C_AVX512/_C_AVX2 or ARM. Selecting one of these classes on non-AMX x86 hardware (or, for the three without any arch check, on ARM) would crash with an AttributeError from inside the kernel call instead of the oracle cleanly rejecting the backend and reporting no supported experts. Add the same x86 + torch.cpu._is_amx_tile_supported() check already used by the linear counterpart (CPUFp8BlockScaledMMKernel.is_supported()) and by check_cpu_sgl_kernel. Tested: tests/kernels/moe/test_cpu_quant_fused_moe.py, tests/kernels/moe/test_cpu_int4_moe.py, tests/kernels/moe/test_cpu_fused_moe.py -- 357 passed, 289 skipped (non-AMX-specific skips), 0 failed on this AMX-capable host, confirming no regression for the eligible case. Signed-off-by: jiang1.li --- .../layers/fused_moe/experts/cpu_moe.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py index ab8a3a3b7a61..b02aef37dda2 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py @@ -508,7 +508,11 @@ def activation_format() -> mk.FusedMoEActivationFormat: @staticmethod def _supports_current_device() -> bool: - return current_platform.is_cpu() + return ( + current_platform.is_cpu() + and current_platform.get_cpu_architecture() == CpuArchEnum.X86 + and torch.cpu._is_amx_tile_supported() + ) @staticmethod def _supports_no_act_and_mul() -> bool: @@ -662,7 +666,11 @@ def activation_format() -> mk.FusedMoEActivationFormat: @staticmethod def _supports_current_device() -> bool: - return current_platform.is_cpu() + return ( + current_platform.is_cpu() + and current_platform.get_cpu_architecture() == CpuArchEnum.X86 + and torch.cpu._is_amx_tile_supported() + ) @staticmethod def _supports_no_act_and_mul() -> bool: @@ -865,7 +873,11 @@ def activation_format() -> mk.FusedMoEActivationFormat: @staticmethod def _supports_current_device() -> bool: - return current_platform.is_cpu() + return ( + current_platform.is_cpu() + and current_platform.get_cpu_architecture() == CpuArchEnum.X86 + and torch.cpu._is_amx_tile_supported() + ) @staticmethod def _supports_no_act_and_mul() -> bool: @@ -1008,6 +1020,7 @@ def _supports_current_device() -> bool: return ( current_platform.is_cpu() and current_platform.get_cpu_architecture() == CpuArchEnum.X86 + and torch.cpu._is_amx_tile_supported() ) @staticmethod From 601e5963506186e70eac8ca316654b123e72a9bc Mon Sep 17 00:00:00 2001 From: "jiang1.li" Date: Mon, 3 Aug 2026 05:07:00 +0000 Subject: [PATCH 5/5] [CPU] Reject misaligned shapes in CPUExpertsInt8 instead of crashing Verified concretely rather than assumed: convert_weight_packed (the shared VNNI prepack all four quantized CPU MoE experts call) requires weight OC % TILE_N(16) == 0 and IC % TILE_K(32) == 0 in csrc/cpu/sgl-kernels/gemm.cpp; moe_int8.cpp additionally hard-requires the intermediate size itself (not 2x) to be a multiple of 32 via its own TORCH_CHECK. Combined across w13 ([E, 2*intermediate, hidden]) and w2 ([E, hidden, intermediate]), both hidden_dim and intermediate_size_per_partition must be multiples of 32 -- exactly the same requirement ArmCPUExpertsInt8 already validates for its NEON kernel, now added to the x86 counterpart. Without this, a misaligned model would pass oracle selection and only fail once process_weights_after_loading calls convert_weight_packed, crashing with a generic "invalid weight out/in features" TORCH_CHECK instead of the oracle cleanly rejecting the backend up front. FP8/MXFP4/INT4 experts are not touched here: convert_weight_packed's own IC accounting doubles for byte-packed uint8 storage (mxfp4/int4), and neither moe_fp8.cpp nor moe_int4.cpp has any runtime shape TORCH_CHECK at all (confirmed by inspection) to cross-validate a threshold against -- getting the modulus wrong for these would either falsely reject working configs or still let bad ones through. Left as a flagged follow-up rather than guessed at. Verified the gate behaves correctly via direct unit check (misaligned hidden_dim/intermediate_size -> rejected with a clear reason; aligned 512/512 -> accepted) and via tests/kernels/moe/test_cpu_quant_fused_moe.py + test_cpu_int4_moe.py + test_cpu_fused_moe.py -- 357 passed, 289 skipped, 0 failed (all existing test shapes are already 32-aligned, so this changes no existing test outcome). Signed-off-by: jiang1.li --- .../layers/fused_moe/experts/cpu_moe.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py index b02aef37dda2..0353d81b3d40 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py @@ -1023,6 +1023,34 @@ def _supports_current_device() -> bool: and torch.cpu._is_amx_tile_supported() ) + @staticmethod + def is_supported_config( + cls: type[mk.FusedMoEExperts], + moe_config: FusedMoEConfig, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + activation_format: mk.FusedMoEActivationFormat, + ) -> tuple[bool, str | None]: + supported, reason = mk.FusedMoEExperts.is_supported_config( + cls, + moe_config, + weight_key, + activation_key, + activation_format, + ) + if not supported: + return supported, reason + # convert_weight_packed (shared VNNI prepack) requires the w13 + # OC/IC and w2 OC/IC to be multiples of TILE_N=16/TILE_K=32; the + # w1 gate-up kernel additionally requires the intermediate size + # itself (not 2x) to be a multiple of 32 (moe_int8.cpp), which + # dominates. Net effect: both dims must be multiples of 32. + if moe_config.hidden_dim % 32 != 0: + return False, "kernel requires hidden dim divisible by 32" + if moe_config.intermediate_size_per_partition % 32 != 0: + return False, "kernel requires intermediate dim divisible by 32" + return True, None + @staticmethod def _supports_no_act_and_mul() -> bool: return False