Skip to content

[Bugfix] Fix Quant Type Descriptor for Weights #32702

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tjtanaa wants to merge 6 commits into
base: main
Choose a base branch
from
Open

[Bugfix] Fix Quant Type Descriptor for Weights #32702

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
34bbc4a
fix the semantics, weight descriptor should use the term channel rath…
tjtanaa Jan 20, 2026
7ead378
add is_per_channel method
tjtanaa Jan 20, 2026
215f1e5
rebase
tjtanaa Jan 21, 2026
307c196
Merge remote-tracking branch 'origin/main' into fix-weight-descriptor
tjtanaa Jan 22, 2026
8783e9d
fix is_per_channel check condition and add documentation
tjtanaa Jan 22, 2026
834a467
add back quant type
tjtanaa Jan 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ...el_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
kFp8DynamicTokenSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kFp8StaticTokenSym,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
cutlass_block_fp8_supported,
Expand All @@ -56,7 +56,7 @@
DYNAMIC_QUANT: kFp8DynamicTokenSym,
}
weight_quant_key_mapping = {
QuantizationStrategy.CHANNEL: kFp8StaticTokenSym,
QuantizationStrategy.CHANNEL: kFp8StaticChannelSym,
QuantizationStrategy.TENSOR: kFp8StaticTensorSym,
}
logger = init_logger(__name__)
Expand Down
4 changes: 2 additions & 2 deletions vllm/model_executor/layers/quantization/fbgemm_fp8.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
from vllm.model_executor.layers.quantization.utils.quant_utils import (
is_layer_skipped,
kFp8DynamicTokenSym,
kFp8StaticTokenSym,
kFp8StaticChannelSym,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
normalize_e4m3fn_to_e4m3fnuz,
Expand Down Expand Up @@ -96,7 +96,7 @@ def __init__(self, quant_config: FBGEMMFp8Config):
self.out_dtype = torch.get_default_dtype()
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=kFp8DynamicTokenSym,
weight_quant_key=kFp8StaticTokenSym,
weight_quant_key=kFp8StaticChannelSym,
out_dtype=torch.get_default_dtype(),
module_name=self.__class__.__name__,
)
Expand Down
4 changes: 2 additions & 2 deletions vllm/model_executor/layers/quantization/modelopt.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@
cutlass_fp4_supported,
is_layer_skipped,
kFp8DynamicTokenSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kFp8StaticTokenSym,
kNvfp4Dynamic,
kNvfp4Static,
swizzle_blockscale,
Expand Down Expand Up @@ -532,7 +532,7 @@ def __init__(self, quant_config: ModelOptFp8Config) -> None:
self.quant_config = quant_config
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=kFp8DynamicTokenSym,
weight_quant_key=kFp8StaticTokenSym,
weight_quant_key=kFp8StaticChannelSym,
out_dtype=torch.get_default_dtype(),
module_name=self.__class__.__name__,
)
Expand Down
4 changes: 2 additions & 2 deletions vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
kFp8DynamicTokenSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kFp8StaticTokenSym,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
normalize_e4m3fn_to_e4m3fnuz,
Expand Down Expand Up @@ -54,7 +54,7 @@ def __init__(
kFp8DynamicTokenSym if per_token_activation else kFp8StaticTensorSym
)
self.weight_quant_key = (
kFp8StaticTokenSym if per_token_weight else kFp8StaticTensorSym
kFp8StaticChannelSym if per_token_weight else kFp8StaticTensorSym
)
self.out_dtype = torch.get_default_dtype()

Expand Down
8 changes: 6 additions & 2 deletions vllm/model_executor/layers/quantization/utils/quant_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,19 @@ def is_per_token(self) -> bool:
return self.row == 1 and self.col == -1

def is_per_channel(self) -> bool:
return self.row == -1 and self.col == 1
return self.row == 1 and self.col == -1

def is_per_group(self) -> bool:
return self.row == 1 and self.col >= 1


GroupShape.PER_TENSOR = GroupShape(-1, -1)
# Input shape is in (M, K)
# Descriptor for weights that are quantized per token
GroupShape.PER_TOKEN = GroupShape(1, -1)
GroupShape.PER_CHANNEL = GroupShape(-1, 1)
Copy link
Collaborator

@ProExpertProg ProExpertProg Jan 22, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we need to just call this one per-row for both? Wdyt @LucasWilkinson @mgoin

# Weight shape is in (N, K)
# Descriptor for weights that are quantized per output channel
GroupShape.PER_CHANNEL = GroupShape(1, -1)


@dataclass(frozen=True)
Expand Down