Repository navigation
[CompressedTensors] FP4 Qutlass Integration #43229
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 8 commits
c2860e2
5b8e54b
3102cf1
78735a7
3106b9b
307a7c1
e7ed3dd
4a478cf
8860208
592adf2
7bccfd3
9d211c7
afc5172
35d515a
2693aaf
e3014f2
20edbcc
0cc46e5
cb7c65f
977e98d
2577ade
90a27e5
b4dc58d
74129b9
851c559
a445925
18ef136
5d31ace
15d9e23
6c39bf4
4a2393e
05da5d2
021d70c
48b794e
1704407
f2882ad
a767e04
870311f
1323134
a144e9a
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||||
|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -2,7 +2,9 @@ | |||||||||
| # SPDX-FileCopyrightText: Copyright contributors to the vLLM project | ||||||||||
|
|
||||||||||
| import torch | ||||||||||
| from torch.nn.parameter import Parameter | ||||||||||
|
|
||||||||||
| from vllm._custom_ops import fusedQuantizeNv | ||||||||||
| from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501 | ||||||||||
| CompressedTensorsScheme, | ||||||||||
| CompressedTensorsW4A4Fp4, | ||||||||||
|
|
@@ -11,9 +13,18 @@ | |||||||||
| CompressedTensorsLinearTransformMethod, | ||||||||||
| TransformTuple, | ||||||||||
| ) | ||||||||||
| from vllm.model_executor.layers.quantization.qutlass_utils import to_blocked | ||||||||||
| from vllm.model_executor.layers.quantization.utils.nvfp4_utils import ( | ||||||||||
| slice_nvfp4_output, | ||||||||||
| ) | ||||||||||
| from vllm.utils.flashinfer import ( | ||||||||||
| flashinfer_scaled_fp4_mm, | ||||||||||
| ) | ||||||||||
|
|
||||||||||
| __all__ = ["is_qutlass_fp4_scheme", "QutlassNvFP4LinearMethod"] | ||||||||||
|
|
||||||||||
| NVFP4_MAX = 6.0 | ||||||||||
|
|
||||||||||
|
|
||||||||||
| def is_qutlass_fp4_scheme( | ||||||||||
| quant_scheme: CompressedTensorsScheme | None, | ||||||||||
|
|
@@ -50,15 +61,69 @@ def create_weights( | |||||||||
| ) | ||||||||||
|
|
||||||||||
| assert self.input_transform is not None | ||||||||||
| assert len(self.input_transform.weight) == 1 | ||||||||||
| assert self.input_transform.weight[0].size(0) == layer.scheme.group_size | ||||||||||
| assert len(self.input_transform.weight.partitions) == 1 | ||||||||||
| assert ( | ||||||||||
| self.input_transform.weight.partitions[0].data.shape[0] | ||||||||||
| == layer.scheme.group_size | ||||||||||
| ) | ||||||||||
|
|
||||||||||
| return ret | ||||||||||
|
|
||||||||||
| def process_weights_after_loading(self, layer): | ||||||||||
| super().process_weights_after_loading(layer) | ||||||||||
|
|
||||||||||
| assert self.input_transform is not None | ||||||||||
| h = self.input_transform.weight.partitions[0].data | ||||||||||
|
kylesayrs marked this conversation as resolved.
Resolved
|
||||||||||
| h_normalized = (h * self.input_transform.scales[0]).to(torch.bfloat16) | ||||||||||
| layer.hadamard_matrix = Parameter(h_normalized, requires_grad=False) | ||||||||||
|
kylesayrs marked this conversation as resolved.
Resolved
kylesayrs marked this conversation as resolved.
Resolved
|
||||||||||
|
|
||||||||||
| # fusedQuantizeNv stores raw absmax as block scales (sf = absmax), | ||||||||||
| # while CT weights use sf = absmax * SFScaleVal / 6.0. The GEMM | ||||||||||
| # computes alpha * sum(fp4_a * sf_a * fp4_w * sf_w), so alpha must | ||||||||||
| # compensate: alpha = weight_global_scale / 6.0 | ||||||||||
| layer.fused_alpha = Parameter( | ||||||||||
| layer.weight_global_scale / NVFP4_MAX, requires_grad=False | ||||||||||
| ) | ||||||||||
|
|
||||||||||
| layer.fused_global_scale = Parameter( | ||||||||||
| torch.tensor([NVFP4_MAX], dtype=torch.float32, device=h.device), | ||||||||||
| requires_grad=False, | ||||||||||
| ) | ||||||||||
|
|
||||||||||
| def apply( | ||||||||||
| self, | ||||||||||
| layer: torch.nn.Module, | ||||||||||
| x: torch.Tensor, | ||||||||||
| bias: torch.Tensor | None = None, | ||||||||||
| ) -> torch.Tensor: | ||||||||||
| raise NotImplementedError() | ||||||||||
| assert bias is None | ||||||||||
| output_size = layer.output_size_per_partition | ||||||||||
| output_shape = [*x.shape[:-1], output_size] | ||||||||||
|
|
||||||||||
| x_flat = x.contiguous().flatten(end_dim=-2) | ||||||||||
|
|
||||||||||
| x_fp4, x_scales = fusedQuantizeNv( | ||||||||||
| x_flat, layer.hadamard_matrix, layer.fused_global_scale | ||||||||||
| ) | ||||||||||
|
|
||||||||||
| x_scales_blocked = to_blocked(x_scales, backend="triton").view(x_scales.shape) | ||||||||||
|
|
||||||||||
| out = flashinfer_scaled_fp4_mm( | ||||||||||
| x_fp4, | ||||||||||
| layer.weight, | ||||||||||
| x_scales_blocked, | ||||||||||
| layer.weight_scale, | ||||||||||
| layer.fused_alpha, | ||||||||||
| x.dtype, | ||||||||||
| backend="cutlass", | ||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think we should probably get the backend to use from
See vllm/vllm/model_executor/kernels/linear/__init__.py Lines 883 to 885 in 251f7e4
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hi @LopezCastroRoberto , please see new commit to infer the flashinfer backend string to use based on the kernel that set. this will serve the same purpose as parsing |
||||||||||
| ) | ||||||||||
|
|
||||||||||
| out = slice_nvfp4_output(out, output_size) | ||||||||||
|
|
||||||||||
| if self.output_transform is not None: | ||||||||||
| for part_id, (start, length) in enumerate(self.partition_ranges): | ||||||||||
| out[:, start : start + length] = self.output_transform( | ||||||||||
| out[:, start : start + length].clone(), part_id=part_id | ||||||||||
| ) | ||||||||||
|
|
||||||||||
| return out.view(*output_shape) | ||||||||||
Uh oh!
There was an error while loading. Please reload this page.