Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
c2860e2
implementation
kylesayrs May 20, 2026
5b8e54b
minor updates
brian-dellabetta Jun 2, 2026
3102cf1
claude compile resolution
brian-dellabetta Jun 2, 2026
78735a7
claude compile resolution p2
brian-dellabetta Jun 2, 2026
3106b9b
switch to flashinfer_scaled_fp4_mm
brian-dellabetta Jun 5, 2026
307a7c1
stylefix
brian-dellabetta Jun 5, 2026
e7ed3dd
quipv16+nvfp4 smoke test
brian-dellabetta Jun 10, 2026
4a478cf
suppress torch warning
brian-dellabetta Jun 10, 2026
8860208
hadacore_transform inplace
brian-dellabetta Jun 10, 2026
592adf2
suppress torch warning for fusedQuantizedNv
brian-dellabetta Jun 10, 2026
7bccfd3
safeFusedQuantize poc
brian-dellabetta Jun 11, 2026
9d211c7
revert safeFusedQuantizeNv, fused working
brian-dellabetta Jun 12, 2026
afc5172
stylefixes
brian-dellabetta Jun 12, 2026
35d515a
safe fusedQuantizeNv
brian-dellabetta Jun 15, 2026
2693aaf
format fix
brian-dellabetta Jun 15, 2026
e3014f2
updates
brian-dellabetta Jun 16, 2026
20edbcc
prune hadacore changes
brian-dellabetta Jun 16, 2026
0cc46e5
remove new line
brian-dellabetta Jun 16, 2026
cb7c65f
merge conflict fix
brian-dellabetta Jun 16, 2026
977e98d
add head_dim constraint
brian-dellabetta Jun 16, 2026
2577ade
add head_dim constraint
brian-dellabetta Jun 16, 2026
90a27e5
Merge branch 'main' into kylesayrs/ct-transforms
brian-dellabetta Jul 10, 2026
b4dc58d
autoawq typo
brian-dellabetta Jul 10, 2026
74129b9
working with fused tensors but awkward
brian-dellabetta Jul 10, 2026
851c559
Merge remote-tracking branch 'origin/main' into kylesayrs/ct-transforms
brian-dellabetta Jul 14, 2026
a445925
cleanup, precision fix
brian-dellabetta Jul 14, 2026
18ef136
Merge branch 'main' into kylesayrs/ct-transforms
brian-dellabetta Jul 14, 2026
5d31ace
inferred gemm backend
brian-dellabetta Jul 16, 2026
15d9e23
formatting
brian-dellabetta Jul 16, 2026
6c39bf4
Merge branch 'main' into kylesayrs/ct-transforms
brian-dellabetta Jul 20, 2026
4a2393e
Merge branch 'main' into kylesayrs/ct-transforms
brian-dellabetta Jul 20, 2026
05da5d2
Merge branch 'main' into kylesayrs/ct-transforms
brian-dellabetta Jul 21, 2026
021d70c
add has device capability check
brian-dellabetta Jul 21, 2026
48b794e
change to 100
brian-dellabetta Jul 21, 2026
1704407
Merge branch 'main' into kylesayrs/ct-transforms
brian-dellabetta Jul 21, 2026
f2882ad
Merge branch 'main' into kylesayrs/ct-transforms
dsikka Jul 22, 2026
a767e04
Merge branch 'main' into kylesayrs/ct-transforms
brian-dellabetta Jul 23, 2026
870311f
Merge branch 'main' into kylesayrs/ct-transforms
brian-dellabetta Jul 24, 2026
1323134
Merge branch 'main' into kylesayrs/ct-transforms
brian-dellabetta Jul 27, 2026
a144e9a
Merge branch 'main' into kylesayrs/ct-transforms
brian-dellabetta Jul 27, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions tests/quantization/test_compressed_tensors.py
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,11 @@ def check_model(model):
"Flat is better than nested.\nSparse is better than dense.",
150.0,
),
(
"nm-testing/Llama-3.2-1B-Instruct-quipv16-nvfp4",
"Flat is better than nested.\nSparse is better than dense.",
150.0,
),
],
)
def test_compressed_tensors_transforms_perplexity(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ def forward(self, value: Tensor, part_id: int = 0) -> Tensor:
if self.transforms[part_id].scheme.head_dim is not None:
weight_size = self.transforms[part_id].scheme.head_dim
value = value.unflatten(-1, (-1, weight_size))
value = ops.hadacore_transform(value)
value = ops.hadacore_transform(value.clone())
Comment thread
kylesayrs marked this conversation as resolved.
Resolved
value = value.flatten(-2, -1)

return value
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import torch
from torch.nn.parameter import Parameter

from vllm._custom_ops import fusedQuantizeNv
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
CompressedTensorsScheme,
CompressedTensorsW4A4Fp4,
Expand All @@ -11,9 +13,18 @@
CompressedTensorsLinearTransformMethod,
TransformTuple,
)
from vllm.model_executor.layers.quantization.qutlass_utils import to_blocked
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
slice_nvfp4_output,
)
from vllm.utils.flashinfer import (
flashinfer_scaled_fp4_mm,
)

__all__ = ["is_qutlass_fp4_scheme", "QutlassNvFP4LinearMethod"]

NVFP4_MAX = 6.0


def is_qutlass_fp4_scheme(
quant_scheme: CompressedTensorsScheme | None,
Expand Down Expand Up @@ -50,15 +61,69 @@ def create_weights(
)

assert self.input_transform is not None
assert len(self.input_transform.weight) == 1
assert self.input_transform.weight[0].size(0) == layer.scheme.group_size
assert len(self.input_transform.weight.partitions) == 1
assert (
self.input_transform.weight.partitions[0].data.shape[0]
== layer.scheme.group_size
)

return ret

def process_weights_after_loading(self, layer):
super().process_weights_after_loading(layer)

assert self.input_transform is not None
h = self.input_transform.weight.partitions[0].data
Comment thread
kylesayrs marked this conversation as resolved.
Resolved
h_normalized = (h * self.input_transform.scales[0]).to(torch.bfloat16)
layer.hadamard_matrix = Parameter(h_normalized, requires_grad=False)
Comment thread
kylesayrs marked this conversation as resolved.
Resolved
Comment thread
kylesayrs marked this conversation as resolved.
Resolved

# fusedQuantizeNv stores raw absmax as block scales (sf = absmax),
# while CT weights use sf = absmax * SFScaleVal / 6.0. The GEMM
# computes alpha * sum(fp4_a * sf_a * fp4_w * sf_w), so alpha must
# compensate: alpha = weight_global_scale / 6.0
layer.fused_alpha = Parameter(
layer.weight_global_scale / NVFP4_MAX, requires_grad=False
)

layer.fused_global_scale = Parameter(
torch.tensor([NVFP4_MAX], dtype=torch.float32, device=h.device),
requires_grad=False,
)

def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
raise NotImplementedError()
assert bias is None
output_size = layer.output_size_per_partition
output_shape = [*x.shape[:-1], output_size]

x_flat = x.contiguous().flatten(end_dim=-2)

x_fp4, x_scales = fusedQuantizeNv(
x_flat, layer.hadamard_matrix, layer.fused_global_scale
)

x_scales_blocked = to_blocked(x_scales, backend="triton").view(x_scales.shape)

out = flashinfer_scaled_fp4_mm(
x_fp4,
layer.weight,
x_scales_blocked,
layer.weight_scale,
layer.fused_alpha,
x.dtype,
backend="cutlass",

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should probably get the backend to use from

_POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
which is expected to be in priority/performance order (when available)

See

def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
"""Select and instantiate the best NVFP4 linear kernel for the
current platform."""

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @LopezCastroRoberto , please see new commit to infer the flashinfer backend string to use based on the kernel that set. this will serve the same purpose as parsing _POSSIBLE_NVFP4_KERNELS

5d31ace

)

out = slice_nvfp4_output(out, output_size)

if self.output_transform is not None:
for part_id, (start, length) in enumerate(self.partition_ranges):
out[:, start : start + length] = self.output_transform(
out[:, start : start + length].clone(), part_id=part_id
)

return out.view(*output_shape)
11 changes: 10 additions & 1 deletion vllm/model_executor/layers/quantization/qutlass_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ def triton_scale_swizzle(
)


@torch.library.custom_op("vllm::triton_mx_block_rearrange", mutates_args=())
def triton_mx_block_rearrange(scale_tensor: torch.Tensor) -> torch.Tensor:
"""
Rearranges an E8M0 tensor scale from row-major format to
Expand Down Expand Up @@ -142,6 +143,14 @@ def triton_mx_block_rearrange(scale_tensor: torch.Tensor) -> torch.Tensor:
return out


@triton_mx_block_rearrange.register_fake
def _triton_mx_block_rearrange_fake(scale_tensor: torch.Tensor) -> torch.Tensor:
rows, cols = scale_tensor.shape
padded_rows = cdiv(rows, 128) * 128
padded_cols = cdiv(cols, 4) * 4
return scale_tensor.new_empty((padded_rows, padded_cols))


def to_blocked(
input_matrix: torch.Tensor, backend: Literal["torch", "triton"] = "triton"
) -> torch.Tensor:
Expand All @@ -157,7 +166,7 @@ def to_blocked(
backend: "torch" (PyTorch path) or "triton" (Triton kernel)

Returns:
Rearranged tensor of shape (32*cdiv(H,128), 16*cdiv(W,4))
Rearranged flattened tensor of size (32*cdiv(H,128) * 16*cdiv(W,4))
"""
if backend == "triton":
return triton_mx_block_rearrange(input_matrix).flatten()
Expand Down
Loading