diff --git a/CMakeLists.txt b/CMakeLists.txt index afbe980e69e4..edfd050b72ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -651,7 +651,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") # Keep building Marlin for 9.0 as there are some group sizes and shapes that # are not supported by Machete yet. - # marlin arches for fp16 output + # marlin arches for fp16/bf16 output # Family-conditional 12.0f (one cubin for SM12x family) requires CUDA >= 13.0; # fall back to architecture-specific 12.0a;12.1a on CUDA < 13.0 (e.g. 12.8). if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) @@ -661,12 +661,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") endif() # marlin has limited support for turing cuda_archs_loose_intersection(MARLIN_SM75_ARCHS "7.5" "${CUDA_ARCHS}") - # marlin arches for bf16 output (we need 9.0 for bf16 atomicAdd PTX) - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) - cuda_archs_loose_intersection(MARLIN_BF16_ARCHS "8.0+PTX;9.0+PTX;12.0f" "${CUDA_ARCHS}") - else() - cuda_archs_loose_intersection(MARLIN_BF16_ARCHS "8.0+PTX;9.0+PTX;12.0a;12.1a" "${CUDA_ARCHS}") - endif() # marlin arches for fp8 input # - sm80 doesn't support fp8 computation # - sm90 and sm100 don't support QMMA.16832.F32.E4M3.E4M3 SAAS instruction @@ -721,7 +715,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") endif() if (MARLIN_ARCHS) - file(GLOB MARLIN_TEMPLATE_KERNEL_SRC "csrc/quantization/marlin/sm80_kernel_*_float16.cu") + file(GLOB MARLIN_TEMPLATE_KERNEL_SRC "csrc/quantization/marlin/sm80_kernel_*.cu") set_gencode_flags_for_srcs( SRCS "${MARLIN_TEMPLATE_KERNEL_SRC}" CUDA_ARCHS "${MARLIN_ARCHS}") @@ -730,16 +724,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") PROPERTIES COMPILE_FLAGS "-static-global-template-stub=false") endif() list(APPEND VLLM_STABLE_EXT_SRC ${MARLIN_TEMPLATE_KERNEL_SRC}) - - file(GLOB MARLIN_TEMPLATE_BF16_KERNEL_SRC "csrc/quantization/marlin/sm80_kernel_*_bfloat16.cu") - set_gencode_flags_for_srcs( - SRCS "${MARLIN_TEMPLATE_BF16_KERNEL_SRC}" - CUDA_ARCHS "${MARLIN_BF16_ARCHS}") - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8) - set_source_files_properties(${MARLIN_TEMPLATE_BF16_KERNEL_SRC} - PROPERTIES COMPILE_FLAGS "-static-global-template-stub=false") - endif() - list(APPEND VLLM_STABLE_EXT_SRC ${MARLIN_TEMPLATE_BF16_KERNEL_SRC}) endif() if (MARLIN_SM75_ARCHS) diff --git a/recipes/models/MiniMaxAI/MiniMax-M3.yaml b/recipes/models/MiniMaxAI/MiniMax-M3.yaml index 096de79bc330..3f9f3a39a401 100644 --- a/recipes/models/MiniMaxAI/MiniMax-M3.yaml +++ b/recipes/models/MiniMaxAI/MiniMax-M3.yaml @@ -185,9 +185,8 @@ variants: - "--max-model-len" - "131072" h200: - # Humming runs the dense MXFP8 linear layers; the atomic-add env var - # enables Marlin's atomic reduction on the MXFP8 MoE path. This image - # also supports the FP8 MSA indexer cache on H200. + # Humming runs the dense MXFP8 linear layers. This image also supports + # the FP8 MSA indexer cache on H200. docker_image: "vllm/vllm-openai:nightly-1a001d584244bd21829a5e5bd4a376ea141c0ce3" extra_args: - "--attention-backend" @@ -199,8 +198,6 @@ variants: - "--gpu-memory-utilization" - "0.95" - "--enable-prefix-caching" - extra_env: - VLLM_MARLIN_USE_ATOMIC_ADD: "1" mi300x: docker_image: "vllm/vllm-openai-rocm:nightly" extra_args: diff --git a/tests/config/test_config_generation.py b/tests/config/test_config_generation.py index eea9e8fa4ab5..60ac34679583 100644 --- a/tests/config/test_config_generation.py +++ b/tests/config/test_config_generation.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import logging + import pytest from vllm.engine.arg_utils import EngineArgs @@ -111,6 +113,15 @@ def test_unrecognized_env(monkeypatch): engine_args.create_engine_config() +def test_deprecated_marlin_atomic_add_env_warns(monkeypatch, caplog_vllm): + monkeypatch.setenv("VLLM_MARLIN_USE_ATOMIC_ADD", "1") + with caplog_vllm.at_level(logging.WARNING, logger="vllm"): + EngineArgs().create_engine_config() + assert any( + "VLLM_MARLIN_USE_ATOMIC_ADD is deprecated" in m for m in caplog_vllm.messages + ) + + def test_get_diff_sampling_param_includes_penalties(): """``--override-generation-config`` must apply presence_penalty and frequency_penalty as server-side defaults, like the other sampling diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 3dc1d1187e3c..68b1862f499a 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -2274,6 +2274,11 @@ def create_engine_config( device_config = DeviceConfig(device=cast(Device, current_platform.device_type)) current_platform.validate_environ(self.fail_on_environ_validation) + if envs.is_set("VLLM_MARLIN_USE_ATOMIC_ADD"): + logger.warning( + "VLLM_MARLIN_USE_ATOMIC_ADD is deprecated, has no effect, and " + "will be removed in v0.35.0." + ) # Check if the model is a speculator and override model/tokenizer/config # BEFORE creating ModelConfig, so the config is created with the target model diff --git a/vllm/envs.py b/vllm/envs.py index ed89f6dcb8dd..a433d1c49805 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -1596,7 +1596,7 @@ def _resolve_rust_cli_path() -> str | None: "VLLM_MODEL_REDIRECT_PATH": lambda: os.environ.get( "VLLM_MODEL_REDIRECT_PATH", None ), - # Whether to use atomicAdd reduce in gptq/awq marlin kernel. + # Deprecated and has no effect; will be removed in v0.35.0. "VLLM_MARLIN_USE_ATOMIC_ADD": lambda: ( os.environ.get("VLLM_MARLIN_USE_ATOMIC_ADD", "0") == "1" ), diff --git a/vllm/model_executor/layers/quantization/utils/marlin_utils.py b/vllm/model_executor/layers/quantization/utils/marlin_utils.py index 1ee52a17802b..9b3e82487004 100644 --- a/vllm/model_executor/layers/quantization/utils/marlin_utils.py +++ b/vllm/model_executor/layers/quantization/utils/marlin_utils.py @@ -635,18 +635,6 @@ def moe_awq_to_marlin_zero_points( return output -def maybe_warn_marlin_atomic_add(device, dtype): - if torch.compiler.is_dynamo_compiling(): - return - device_capability = torch.cuda.get_device_capability(device) - if device_capability[0] < 9 and dtype == torch.bfloat16: - logger.info_once( - "You are running Marlin kernel with bf16 on GPUs before SM90. " - "You can consider change to fp16 to achieve better performance " - "if possible." - ) - - def moe_packed_to_marlin_zero_points( q_zp_packed: torch.Tensor, size_k: int, @@ -671,42 +659,6 @@ def moe_packed_to_marlin_zero_points( return output -def maybe_warn_marlin_atomic_add_env(): - if torch.compiler.is_dynamo_compiling(): - return - if envs.VLLM_MARLIN_USE_ATOMIC_ADD: - return - logger.info_once( - "Marlin kernel can achieve better performance for small size_n " - "with experimental use_atomic_add feature. " - "You can consider set environment variable " - "VLLM_MARLIN_USE_ATOMIC_ADD to 1 if possible." - ) - - -def should_use_atomic_add_reduce( - m: int, n: int, k: int, device: torch.device, dtype: torch.dtype -) -> bool: - # the performance of atomicAdd is better than global reduce - # only when m*n is small and k is large - if n >= 2048 or k < 2048 or device.type != "cuda": - return False - - # disable atomicAdd reduce by default, - # one can enable it with VLLM_MARLIN_USE_ATOMIC_ADD=1 - if not envs.VLLM_MARLIN_USE_ATOMIC_ADD: - maybe_warn_marlin_atomic_add_env() - return False - - # sm8x doesn't support atomicAdd + bfloat16 natively - device_capability = torch.cuda.get_device_capability(device) - if device_capability[0] < 9 and dtype == torch.bfloat16: - maybe_warn_marlin_atomic_add(device, dtype) - return False - - return True - - _quant_fp8_method: QuantFP8 | None = None @@ -769,14 +721,6 @@ def apply_gptq_marlin_linear( padded_n, padded_k = marlin_repacked_nk(weight, wtype.size_bits) reshaped_x = marlin_pad_dim(reshaped_x, input_size_per_partition, padded_k) - use_atomic_add = should_use_atomic_add_reduce( - m=reshaped_x.size(0), - n=padded_n, - k=padded_k, - device=input.device, - dtype=input.dtype, - ) - a_scales = None if input_dtype == torch.int8: assert wtype == scalar_types.uint4b8, ( @@ -804,7 +748,7 @@ def apply_gptq_marlin_linear( size_m=reshaped_x.shape[0], size_n=padded_n, size_k=padded_k, - use_atomic_add=use_atomic_add, + use_atomic_add=False, use_fp32_reduce=use_fp32_reduce, is_zp_float=False, ) diff --git a/vllm/model_executor/layers/quantization/utils/marlin_utils_fp4.py b/vllm/model_executor/layers/quantization/utils/marlin_utils_fp4.py index b0fae4521a9f..fb600590fe44 100644 --- a/vllm/model_executor/layers/quantization/utils/marlin_utils_fp4.py +++ b/vllm/model_executor/layers/quantization/utils/marlin_utils_fp4.py @@ -19,7 +19,6 @@ marlin_quant_input, marlin_repacked_nk, marlin_unpad_output, - should_use_atomic_add_reduce, ) from vllm.model_executor.utils import replace_parameter from vllm.platforms import current_platform @@ -177,14 +176,6 @@ def apply_fp4_marlin_linear( padded_n, padded_k = marlin_repacked_nk(weight, num_bits=4) reshaped_x = marlin_pad_dim(reshaped_x, size_k, padded_k) - use_atomic_add = should_use_atomic_add_reduce( - m=reshaped_x.size(0), - n=padded_n, - k=padded_k, - device=input.device, - dtype=input.dtype, - ) - inputs = reshaped_x a_scales = None is_nvfp4 = weight_global_scale is not None @@ -209,7 +200,7 @@ def apply_fp4_marlin_linear( size_m=reshaped_x.size(0), size_n=padded_n, size_k=padded_k, - use_atomic_add=use_atomic_add, + use_atomic_add=False, use_fp32_reduce=use_fp32_reduce, ) diff --git a/vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py b/vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py index 257c5bf44505..490a3b325ada 100644 --- a/vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py +++ b/vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py @@ -18,7 +18,6 @@ marlin_permute_scales, marlin_repacked_nk, marlin_unpad_output, - should_use_atomic_add_reduce, ) from vllm.model_executor.utils import replace_parameter from vllm.platforms import current_platform @@ -65,14 +64,6 @@ def apply_fp8_marlin_linear( padded_n, padded_k = marlin_repacked_nk(weight, num_bits=8) reshaped_x = marlin_pad_dim(reshaped_x, size_k, padded_k) - use_atomic_add = should_use_atomic_add_reduce( - m=reshaped_x.size(0), - n=padded_n, - k=padded_k, - device=input.device, - dtype=input.dtype, - ) - inputs = reshaped_x a_scales = None if input_dtype is not None and input_dtype.itemsize == 1: @@ -92,7 +83,7 @@ def apply_fp8_marlin_linear( size_m=reshaped_x.size(0), size_n=padded_n, size_k=padded_k, - use_atomic_add=use_atomic_add, + use_atomic_add=False, use_fp32_reduce=use_fp32_reduce, ) @@ -388,14 +379,6 @@ def apply_mxfp8_marlin_linear( padded_n, padded_k = marlin_repacked_nk(weight, num_bits=8) reshaped_x = marlin_pad_dim(reshaped_x, size_k, padded_k) - use_atomic_add = should_use_atomic_add_reduce( - m=reshaped_x.size(0), - n=padded_n, - k=padded_k, - device=input.device, - dtype=input.dtype, - ) - output = torch.ops.vllm.marlin_gemm( a=reshaped_x, b_q_weight=weight, @@ -409,7 +392,7 @@ def apply_mxfp8_marlin_linear( size_m=reshaped_x.size(0), size_n=padded_n, size_k=padded_k, - use_atomic_add=use_atomic_add, + use_atomic_add=False, use_fp32_reduce=use_fp32_reduce, )