Repository navigation
GEMM+RoPE+MXFP8 fusion #367
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Anerudhan
merged 6 commits into
NVIDIA:develop
from
chaseblock:cblock/gemm_rope_mxfp8_fusion
Jul 10, 2026
Merged
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit
Hold shift + click to select a range
a39da3f
Add fused gemm+rope+mxfp8quant kernel.
chaseblock 19005d9
Merge branch 'NVIDIA:develop' into cblock/gemm_rope_mxfp8_fusion
chaseblock e58218d
Add documentation and general interface names
chaseblock f1575d6
Address coderabbitai's suggestions
chaseblock e35055a
Additional tests for fused gemm+rope+mxfp8
chaseblock 2c89b1e
Remove NUM_HEADS constant
chaseblock File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Some comments aren't visible on the classic Files Changed page.
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| from .api import ( | ||
| GemmProjRopeMxfp8Sm100, | ||
| gemm_proj_rope_mxfp8_wrapper_sm100, | ||
| ) | ||
| from .gemm_proj_rope_mxfp8 import ( | ||
| run, | ||
| gemm_proj_rope_mxfp8_reference, | ||
| ) | ||
|
|
||
| __all__ = [ | ||
| "GemmProjRopeMxfp8Sm100", | ||
| "gemm_proj_rope_mxfp8_wrapper_sm100", | ||
| "run", | ||
| "gemm_proj_rope_mxfp8_reference", | ||
| ] | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,246 @@ | ||
| from .gemm_proj_rope_mxfp8 import ( | ||
| gemm_proj_rope_mxfp8_host, | ||
| NUM_HEADS, | ||
| HEAD_DIM, | ||
| BLOCK, | ||
| TILE_M, | ||
| ) | ||
|
|
||
| from cuda.bindings import driver as cuda | ||
| import logging | ||
| import torch | ||
| from typing import Optional | ||
|
|
||
| import cutlass | ||
| import cutlass.utils | ||
| import cutlass.cute as cute | ||
| from cutlass.cute.runtime import from_dlpack | ||
|
|
||
| from cudnn.api_base import APIBase, TupleDict | ||
|
|
||
|
|
||
| class GemmProjRopeMxfp8Sm100(APIBase): | ||
| """Fused projection GEMM + per-head YARN RoPE + dual-direction MXFP8 quantize (SM100). | ||
| """ | ||
|
|
||
| def __init__( | ||
| self, | ||
| sample_x: torch.Tensor, | ||
| sample_w: torch.Tensor, | ||
| sample_cos: torch.Tensor, | ||
| sample_sin: torch.Tensor, | ||
| sample_q_fp8_row: torch.Tensor, | ||
| sample_q_scales_row: torch.Tensor, | ||
| sample_q_fp8_col: torch.Tensor, | ||
| sample_q_scales_col: torch.Tensor, | ||
| w_out_in: bool = False, | ||
| ): | ||
| super().__init__() | ||
|
|
||
| self._warn_experimental_api() | ||
| self._logger.debug("Entering __init__") | ||
|
|
||
| self.x_desc = self._make_tensor_desc(sample_x, name="sample_x") | ||
| self.w_desc = self._make_tensor_desc(sample_w, name="sample_w") | ||
| self.cos_desc = self._make_tensor_desc(sample_cos, name="sample_cos") | ||
| self.sin_desc = self._make_tensor_desc(sample_sin, name="sample_sin") | ||
| self.q_fp8_row_desc = self._make_tensor_desc(sample_q_fp8_row, name="sample_q_fp8_row") | ||
| self.q_scales_row_desc = self._make_tensor_desc(sample_q_scales_row, name="sample_q_scales_row") | ||
| self.q_fp8_col_desc = self._make_tensor_desc(sample_q_fp8_col, name="sample_q_fp8_col") | ||
| self.q_scales_col_desc = self._make_tensor_desc(sample_q_scales_col, name="sample_q_scales_col") | ||
|
|
||
| self.w_out_in = bool(w_out_in) | ||
| self.tokens = int(sample_x.shape[0]) | ||
|
|
||
| # The cute program is traced from real sample tensors at compile() time; kept only | ||
| # until then, then released (mirrors the sample_* teardown in the sibling APIs). | ||
| self._samples = ( | ||
| sample_x, sample_w, sample_cos, sample_sin, | ||
| sample_q_fp8_row, sample_q_scales_row, sample_q_fp8_col, sample_q_scales_col, | ||
| ) | ||
| self._logger.debug( | ||
| f"__init__ completed: x {self.x_desc.shape}, w {self.w_desc.shape}, " | ||
| f"w_out_in {self.w_out_in}" | ||
| ) | ||
|
|
||
| def check_support(self) -> bool: | ||
| self._logger.debug("Entering check_support") | ||
|
|
||
| self._check_dtype(self.x_desc, dtype=torch.bfloat16, name="x") | ||
| self._check_dtype(self.w_desc, dtype=torch.bfloat16, name="w") | ||
| self._check_dtype(self.cos_desc, dtype=torch.bfloat16, name="cos") | ||
| self._check_dtype(self.sin_desc, dtype=torch.bfloat16, name="sin") | ||
| self._check_dtype(self.q_fp8_row_desc, dtype=torch.float8_e4m3fn, name="q_fp8_row") | ||
| self._check_dtype(self.q_fp8_col_desc, dtype=torch.float8_e4m3fn, name="q_fp8_col") | ||
| self._check_dtype(self.q_scales_row_desc, dtype=torch.uint8, name="q_scales_row") | ||
| self._check_dtype(self.q_scales_col_desc, dtype=torch.uint8, name="q_scales_col") | ||
|
|
||
| self._value_error_if( | ||
| self.tokens % TILE_M != 0, | ||
| f"tokens ({self.tokens}) must be a multiple of TILE_M ({TILE_M})", | ||
| ) | ||
|
|
||
| self._logger.debug("Checking environment") | ||
| self._runtime_error_if(not torch.cuda.is_available(), "CUDA is not available") | ||
| device = torch.cuda.current_device() | ||
| major, minor = torch.cuda.get_device_capability(device) | ||
| compute_capability = major * 10 + minor | ||
| self._runtime_error_if( | ||
| compute_capability < 100, | ||
| f"GemmProjRopeMxfp8 requires SM100+ compute capability, but found " | ||
| f"SM{compute_capability} on device {device}", | ||
| ) | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
|
|
||
| self._is_supported = True | ||
| self._logger.debug("check_support completed successfully") | ||
| return True | ||
|
|
||
| def _to_cute_tensors( | ||
| self, | ||
| x, w, cos, sin, q_fp8_row, q_scales_row, q_fp8_col, q_scales_col, | ||
| ): | ||
| """Wrap the torch tensors as (layout-dynamic) cute tensors for the host program. | ||
|
|
||
| ``w`` may be stored [in, out] (default) or the TE-native [out, in] (``w_out_in``); | ||
| both are presented to the kernel as the logical [out, in] B operand. | ||
| """ | ||
| mA = from_dlpack(x.detach(), assumed_align=16).mark_layout_dynamic(leading_dim=1) | ||
| if self.w_out_in: | ||
| mB = from_dlpack(w.detach(), assumed_align=16).mark_layout_dynamic(leading_dim=1) | ||
| else: | ||
| mB = from_dlpack(w.detach().transpose(0, 1), assumed_align=16).mark_layout_dynamic(leading_dim=0) | ||
| mCos = from_dlpack(cos.detach(), assumed_align=16).mark_layout_dynamic(leading_dim=1) | ||
| mSin = from_dlpack(sin.detach(), assumed_align=16).mark_layout_dynamic(leading_dim=1) | ||
| mQrow = from_dlpack(q_fp8_row, assumed_align=16).mark_layout_dynamic(leading_dim=2) | ||
| mSrow = from_dlpack(q_scales_row, assumed_align=16).mark_layout_dynamic(leading_dim=2) | ||
| mQcol = from_dlpack(q_fp8_col, assumed_align=16).mark_layout_dynamic(leading_dim=2) | ||
| mScol = from_dlpack(q_scales_col, assumed_align=16).mark_layout_dynamic(leading_dim=2) | ||
| return mA, mB, mCos, mSin, mQrow, mSrow, mQcol, mScol | ||
|
|
||
| def compile(self) -> None: | ||
| self._logger.debug("Entering compile") | ||
| self._ensure_support_checked() | ||
| if self._compiled_kernel is not None: | ||
| self._logger.debug("Kernel already compiled; skipping recompilation") | ||
| return | ||
|
|
||
| cute_tensors = self._to_cute_tensors(*self._samples) | ||
| grid_m = self.tokens // TILE_M | ||
| hardware_info = cutlass.utils.HardwareInfo() | ||
| max_active_clusters = hardware_info.get_max_active_clusters(1) | ||
| swizzle_size = 8 | ||
| # Trace/compile on the current stream (a runtime argument; execute() supplies its own). | ||
| compile_stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) | ||
|
|
||
| self._logger.debug("Compiling gemm_proj_rope_mxfp8_host") | ||
| self._compiled_kernel = cute.compile( | ||
| gemm_proj_rope_mxfp8_host, | ||
| *cute_tensors, | ||
| grid_m, | ||
| max_active_clusters, | ||
| swizzle_size, | ||
| compile_stream, | ||
| ) | ||
|
|
||
| self._samples = None | ||
| self._logger.debug("Kernel compiled successfully") | ||
|
|
||
| def execute( | ||
| self, | ||
| x: torch.Tensor, | ||
| w: torch.Tensor, | ||
| cos: torch.Tensor, | ||
| sin: torch.Tensor, | ||
| q_fp8_row: torch.Tensor, | ||
| q_scales_row: torch.Tensor, | ||
| q_fp8_col: torch.Tensor, | ||
| q_scales_col: torch.Tensor, | ||
| current_stream: Optional[cuda.CUstream] = None, | ||
| ) -> None: | ||
| self._logger.debug("Entering execute") | ||
| current_stream = self._get_default_stream(current_stream) | ||
|
|
||
| self._runtime_error_if( | ||
| self._compiled_kernel is None, | ||
| "GemmProjRopeMxfp8Sm100 kernel not compiled; call compile() first", | ||
| ) | ||
|
|
||
| cute_tensors = self._to_cute_tensors( | ||
| x, w, cos, sin, q_fp8_row, q_scales_row, q_fp8_col, q_scales_col | ||
| ) | ||
| self._compiled_kernel(*cute_tensors, current_stream) | ||
| self._logger.debug("Executed with compiled kernel successfully") | ||
|
|
||
|
|
||
| _logger = logging.getLogger(__name__) | ||
| _cache_of_GemmProjRopeMxfp8Sm100Objects = {} | ||
|
|
||
|
|
||
| def gemm_proj_rope_mxfp8_wrapper_sm100( | ||
| x: torch.Tensor, | ||
| w: torch.Tensor, | ||
| cos: torch.Tensor, | ||
| sin: torch.Tensor, | ||
| w_out_in: bool = False, | ||
| stream: Optional[cuda.CUstream] = None, | ||
| ) -> TupleDict: | ||
| """Allocate outputs, (compile-and-)cache the kernel, run it, and return the MXFP8 Q tensors. | ||
|
|
||
| Args: | ||
| x: ``[tokens, Q_LORA]`` bf16 activations (``tokens % TILE_M == 0``). | ||
| w: projection weight -- ``[Q_LORA, NUM_HEADS*HEAD_DIM]`` (``w_out_in=False``) or the | ||
| TE-native transposed ``[NUM_HEADS*HEAD_DIM, Q_LORA]`` (``w_out_in=True``). | ||
| cos, sin: ``[tokens, QK_ROPE]`` bf16 rotary tables. | ||
| w_out_in: whether ``w`` is stored ``[out, in]``. | ||
| stream: optional CUDA stream; defaults to the current torch stream. | ||
|
|
||
| Returns: | ||
| ``TupleDict(q_fp8_row, q_scales_row, q_fp8_col, q_scales_col)`` -- rowwise / columnwise | ||
| MXFP8 (E4M3) data and E8M0 scales. | ||
| """ | ||
| tokens = x.shape[0] | ||
| device = x.device | ||
|
|
||
| q_fp8_row = torch.empty(tokens, NUM_HEADS, HEAD_DIM, dtype=torch.float8_e4m3fn, device=device) | ||
| q_scales_row = torch.empty(tokens, NUM_HEADS, HEAD_DIM // BLOCK, dtype=torch.uint8, device=device) | ||
| q_fp8_col = torch.empty(tokens, NUM_HEADS, HEAD_DIM, dtype=torch.float8_e4m3fn, device=device) | ||
| q_scales_col = torch.empty(tokens // BLOCK, NUM_HEADS, HEAD_DIM, dtype=torch.uint8, device=device) | ||
|
|
||
| cache_key = ( | ||
| tuple(x.shape), | ||
| tuple(w.shape), | ||
| x.dtype, | ||
| w.dtype, | ||
| cos.dtype, | ||
| bool(w_out_in), | ||
| ) | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
| if cache_key in _cache_of_GemmProjRopeMxfp8Sm100Objects: | ||
| _logger.debug("gemm_proj_rope_mxfp8_wrapper_sm100: using cached GemmProjRopeMxfp8Sm100 object") | ||
| obj = _cache_of_GemmProjRopeMxfp8Sm100Objects[cache_key] | ||
| else: | ||
| _logger.debug("gemm_proj_rope_mxfp8_wrapper_sm100: creating new GemmProjRopeMxfp8Sm100 object") | ||
| obj = GemmProjRopeMxfp8Sm100( | ||
| sample_x=x, | ||
| sample_w=w, | ||
| sample_cos=cos, | ||
| sample_sin=sin, | ||
| sample_q_fp8_row=q_fp8_row, | ||
| sample_q_scales_row=q_scales_row, | ||
| sample_q_fp8_col=q_fp8_col, | ||
| sample_q_scales_col=q_scales_col, | ||
| w_out_in=w_out_in, | ||
| ) | ||
| assert obj.check_support() | ||
| obj.compile() | ||
| _cache_of_GemmProjRopeMxfp8Sm100Objects[cache_key] = obj | ||
|
|
||
| obj.execute( | ||
| x, w, cos, sin, q_fp8_row, q_scales_row, q_fp8_col, q_scales_col, current_stream=stream | ||
| ) | ||
|
|
||
| return TupleDict( | ||
| q_fp8_row=q_fp8_row, | ||
| q_scales_row=q_scales_row, | ||
| q_fp8_col=q_fp8_col, | ||
| q_scales_col=q_scales_col, | ||
| ) | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.