diff --git a/python/cudnn/sdpa/fwd/api_dsl.py b/python/cudnn/sdpa/fwd/api_dsl.py index 85ba57e6ce..02e718faee 100644 --- a/python/cudnn/sdpa/fwd/api_dsl.py +++ b/python/cudnn/sdpa/fwd/api_dsl.py @@ -538,9 +538,13 @@ def _dummy(self, key: str, device: torch.device, factory: Callable[[], torch.Ten def _checked_lse_view(self, lse_tensor: torch.Tensor) -> torch.Tensor: """Validate a caller-provided LSE buffer and return the kernel's (B, H_q, S_q) view. - The kernel WRITES through the returned view, so this must be a true - view: a silent ``reshape`` copy of a non-contiguous buffer would - receive the output and be dropped, leaving the caller's LSE unwritten. + The logical contract is exactly ``B*H_q*S_q`` fp32 elements. Graph + Stats commonly arrive as a rank-4 ``(B, H_q, S_q, 1)`` view, which is + reinterpreted as the declared rank-3 LSE layout without copying. The + kernel writes through the returned view, so a silent ``reshape`` copy + of a non-contiguous buffer would leave the caller's Stats unwritten. + Dense adapters therefore record ``_lse_stride`` and rebuild that + declared view directly over the caller's storage. """ self._value_error_if( dtype_name(lse_tensor) != "float32", @@ -551,11 +555,22 @@ def _checked_lse_view(self, lse_tensor: torch.Tensor) -> torch.Tensor: lse_tensor.numel() != expected, f"lse_tensor must have B*H_q*S_q = {expected} elements; got {lse_tensor.numel()}", ) - self._value_error_if( - not lse_tensor.is_contiguous(), - "lse_tensor must be contiguous (the kernel writes through this buffer)", - ) - return lse_tensor.view(self.batch_size, self.h_q, self.s_q_max) + shape = (self.batch_size, self.h_q, self.s_q_max) + stride = getattr(self, "_lse_stride", None) + if stride is None: + self._value_error_if( + not lse_tensor.is_contiguous(), + "lse_tensor must be contiguous (the kernel writes through this buffer)", + ) + return lse_tensor.view(shape) + if tuple(lse_tensor.shape) == shape and tuple(lse_tensor.stride()) == stride: + return lse_tensor + try: + return lse_tensor.as_strided(shape, stride, lse_tensor.storage_offset()) + except RuntimeError as exc: + raise ValueError( + f"lse_tensor backing storage is too small for declared shape {shape}, stride {stride}, and storage_offset {lse_tensor.storage_offset()}" + ) from exc def _checked_sinks_1d(self, sinks: torch.Tensor) -> torch.Tensor: """Validate caller-provided sink logits and return the kernel's (H_q,) fp32 view. @@ -728,6 +743,7 @@ def _initialize_implementation(self) -> None: self.flavor: Optional[tuple[int, int]] = None self.thd_stats_head_major = False self.thd_stats_head_stride = 0 + self._lse_stride: Optional[tuple[int, int, int]] = None self._k_mod = None def check_support(self) -> bool: @@ -853,7 +869,12 @@ def check_support(self) -> bool: self.thd_stats_head_major = head_major self.thd_stats_head_stride = int(stride_h) if head_major else 0 else: - self._value_error_if(not self.lse_desc.is_contiguous(), "LSE must be contiguous on SM100 DSL") + self._value_error_if( + not dense_layout_ok((*self.lse_desc.shape, 1), (*self.lse_desc.stride, 1)), + f"LSE must use a dense-compatible B/H/S permutation or padded layout " + f"with non-broadcast, non-overlapping-by-span strides; got {self.lse_desc.stride}", + ) + self._lse_stride = None if self.lse_desc.is_contiguous() else tuple(int(stride) for stride in self.lse_desc.stride) self._value_error_if(not torch.cuda.is_available(), "CUDA must be available for SM100 DSL SDPA") device = self.q_desc.device @@ -1128,6 +1149,7 @@ def compile(self) -> None: sq=self.s_q_max, skv=self.s_k_max, has_lse=(self.lse_desc is not None) or self.split_kv > 1, + lse_stride=None if self.split_kv > 1 else self._lse_stride, ) if self._pertensor: # ENVELOPE (per-tensor only): hand the kernel the ACTUAL head @@ -1154,6 +1176,7 @@ def compile(self) -> None: d_qk=self.head_dim_qk, d_v=self.head_dim_v, has_lse=(self.lse_desc is not None) or self.split_kv > 1, + lse_stride=None if self.split_kv > 1 else self._lse_stride, ) self._combine_kernel = None if self.split_kv > 1: @@ -1173,6 +1196,7 @@ def compile(self) -> None: dtype_o=self._combine_dtype_tag(), has_lse=self.lse_desc is not None, has_amax=self._fp8, + lse_stride=self._lse_stride, ) self._logger.debug("compile completed") @@ -1378,7 +1402,7 @@ def execute( import cutlass o_arg = O_scratch if o_needs_copy_back else O_view - lse_arg = lse_tensor.reshape(self.batch_size, self.h_q, self.s_q_max) if lse_tensor is not None else None + lse_arg = lse_tensor if self.split_kv > 1: # Redirect the mainloop into split-major partial slabs, then reduce # into the caller's O/LSE with the plan-time-compiled combine pass. @@ -1856,7 +1880,7 @@ def _execute_mxfp8( sf_v_v = self._reshape_sf(sf_v, h_kv, n_kv_tiles, km.SF_SMEM_SIZE_V) # has_lse=False (no Stats output): the store is compiled out; bind None. - lse = lse_tensor.reshape(b, h_q, sq) if lse_tensor is not None else None + lse = lse_tensor sinks_t = ( self._checked_sinks_1d(sinks) if sinks is not None else self._dummy("sinks", device, lambda: torch.zeros(h_q, dtype=torch.float32, device=device)) ) @@ -2023,7 +2047,7 @@ def _execute_fp8( O = O_scratch if o_needs_copy_back else O_view # has_lse=False (no Stats output): the store is compiled out; bind None. - lse = lse_tensor.reshape(b, h_q, sq) if lse_tensor is not None else None + lse = lse_tensor sinks_t = ( self._checked_sinks_1d(sinks) if sinks is not None else self._dummy("sinks", device, lambda: torch.zeros(h_q, dtype=torch.float32, device=device)) ) @@ -2318,6 +2342,7 @@ def _initialize_implementation(self) -> None: self.head_dim_v: Optional[int] = None self.thd_stats_head_major = False self.thd_stats_head_stride = 0 + self._lse_stride: Optional[tuple[int, int, int]] = None self._k_mod = None def check_support(self) -> bool: @@ -2390,7 +2415,12 @@ def check_support(self) -> bool: self.thd_stats_head_major = head_major self.thd_stats_head_stride = int(stride_h) if head_major else 0 else: - self._value_error_if(not self.lse_desc.is_contiguous(), "LSE must be contiguous on SM120 DSL") + self._value_error_if( + not dense_layout_ok((*self.lse_desc.shape, 1), (*self.lse_desc.stride, 1)), + f"LSE must use a dense-compatible B/H/S permutation or padded layout " + f"with non-broadcast, non-overlapping-by-span strides; got {self.lse_desc.stride}", + ) + self._lse_stride = None if self.lse_desc.is_contiguous() else tuple(int(stride) for stride in self.lse_desc.stride) for label, val in ( ("B", b), @@ -2621,6 +2651,7 @@ def compile(self) -> None: # binds no LSE buffer at all (no dummy, no allocation). A split # REQUIRES it: the per-split LSE is the combine weight. has_lse=(self.lse_desc is not None) or self.split_kv > 1, + lse_stride=None if self.split_kv > 1 else self._lse_stride, ) self._combine_kernel = None if self.split_kv > 1: @@ -2638,6 +2669,7 @@ def compile(self) -> None: dtype_o=self._combine_dtype_tag(), has_lse=self.lse_desc is not None, has_amax=False, + lse_stride=self._lse_stride, ) self._logger.debug("compile completed") @@ -3529,6 +3561,7 @@ def _initialize_implementation(self) -> None: self.right_bound_runtime: int = 0 self._k_mod = None self._params = None + self._lse_stride: Optional[tuple[int, int, int]] = None # ------------------------------------------------------------------ def check_support(self) -> bool: @@ -3590,7 +3623,12 @@ def check_support(self) -> bool: if self.lse_desc is not None: self._check_dtype(self.lse_desc, torch.float32, name="LSE") self._check_tensor_shape(self.lse_desc, (b, h_qo, s_qo), name="LSE") - self._value_error_if(not self.lse_desc.is_contiguous(), "LSE must be contiguous on SM80") + self._value_error_if( + not dense_layout_ok((*self.lse_desc.shape, 1), (*self.lse_desc.stride, 1)), + f"LSE must use a dense-compatible B/H/S permutation or padded layout " + f"with non-broadcast, non-overlapping-by-span strides; got {self.lse_desc.stride}", + ) + self._lse_stride = None if self.lse_desc.is_contiguous() else tuple(int(stride) for stride in self.lse_desc.stride) self._not_implemented_error_if( self.thd or self.cu_seq_q_lens or self.cu_seq_kv_lens, @@ -3738,6 +3776,7 @@ def compile(self) -> None: d=self.head_dim_qk, swa_window=int(self.swa_window_runtime), rope_max_s=self._rope_max_s, + lse_stride=self._lse_stride, ) self._logger.debug("compile completed") @@ -3768,8 +3807,6 @@ def execute( # Init-time flags are compile-time specializations; execute must match # them exactly, in both directions (Hard Rule 1). - if lse_tensor is not None and lse_tensor.ndim == 4: - lse_tensor = lse_tensor.squeeze(-1) self._value_error_if(p.has_lse and lse_tensor is None, "compiled with a Stats output but execute() got no lse_tensor") self._value_error_if(not p.has_lse and lse_tensor is not None, "lse_tensor provided but the plan compiled the LSE store out") self._value_error_if(p.has_bias != (bias_tensor is not None), "bias presence must match the compiled specialization") @@ -3777,6 +3814,10 @@ def execute( self._value_error_if(p.has_sink != (sinks is not None), "sinks presence must match the compiled specialization") self._value_error_if(p.has_seq_kv_lens != (seq_kv_lens is not None), "seq_kv_lens presence must match the compiled specialization") self._value_error_if(p.has_seq_q_lens != (seq_q_lens is not None), "seq_q_lens presence must match the compiled specialization") + # Graph Stats declarations arrive as (B, H, S, 1); the kernels write + # [B, H, SQ] through the exact declared strides. + if lse_tensor is not None: + lse_tensor = self._checked_lse_view(lse_tensor) scale_val = self.scale_softmax if (scale_softmax is None or scale_softmax == 0.0) else float(scale_softmax) device = q_tensor.device diff --git a/python/cudnn/sdpa/fwd/engines.py b/python/cudnn/sdpa/fwd/engines.py index 1acf50d468..f28baf9030 100644 --- a/python/cudnn/sdpa/fwd/engines.py +++ b/python/cudnn/sdpa/fwd/engines.py @@ -454,6 +454,20 @@ def mismatch(capabilities: Capabilities, facts: "ga.SdpaGraphFacts", knobs: Opti if facts.padded and facts.wants_stats and not facts.thd and not capabilities.padded_stats: return "padding mask with generate_stats is not supported yet (per-batch seq_len_q LSE trim not plumbed)" + if facts.stats_t is not None and not facts.thd: + if facts.stats_t.get_data_type() != cudnn.data_type.FLOAT: + return f"stats must be fp32; got {facts.stats_t.get_data_type()}" + stats_dim = tuple(facts.stats_t.get_dim()) + stats_stride = tuple(facts.stats_t.get_stride()) + expected_dim = (facts.b, facts.h_q, facts.s_q, 1) + if stats_dim != expected_dim: + return f"stats must be (B, H_q, S_q, 1) = {expected_dim}; got {stats_dim}" + if not ga.dense_layout_ok(stats_dim, stats_stride): + return ( + "stats must use a dense-compatible B/H/S permutation or padded layout " + f"with non-broadcast, non-overlapping-by-span strides; got {stats_stride}" + ) + if capabilities.single_wave_only: # See Capabilities.single_wave_only. 512 = TILES_Q * TILE_M * CTA_MMA # rows per cluster; resident clusters = SM count / CTA_MMA (one CTA per diff --git a/python/cudnn/sdpa/fwd/kernels/prefill_d128_f16_sm100.py b/python/cudnn/sdpa/fwd/kernels/prefill_d128_f16_sm100.py index e8c29b3c23..0628a598ec 100644 --- a/python/cudnn/sdpa/fwd/kernels/prefill_d128_f16_sm100.py +++ b/python/cudnn/sdpa/fwd/kernels/prefill_d128_f16_sm100.py @@ -1987,8 +1987,7 @@ def _correction_warp_group( # the chunk's O. The pair (O_s, lse_s) is everything the # combine needs. Folds to batch_idx at SPLIT_KV == 1. lse_batch = _partial_batch(batch_idx, split_idx, n_batch) - lse_row = lse_arr[lse_batch, row_head_idx, :] - lse_row[q_row_global] = lse_val + lse_arr[lse_batch, row_head_idx, q_row_global] = lse_val sO_sub_base = sO[qs].base @@ -2233,6 +2232,7 @@ def compile( # noqa: A001 k_stride: Optional[tuple] = None, v_stride: Optional[tuple] = None, o_stride: Optional[tuple] = None, + lse_stride: Optional[tuple[int, int, int]] = None, ) -> Callable: """Compile a kernel with ALL dims concrete to pin TMA descriptor strides at compile time. @@ -2270,6 +2270,8 @@ def compile( # noqa: A001 # Each split's LSE is not optional under KV split — it IS the weight the # combine reduces with. Without it the partials cannot be recombined. raise ValueError("d128: split_kv > 1 requires has_lse=True (the per-split LSE drives the combine)") + if lse_stride is not None and (CFG.THD_VARLEN or SPLIT_KV > 1): + raise ValueError("dense LSE strides are not valid for THD or split-KV workspaces") _fake_batch = 1 if CFG.THD_VARLEN else b if CFG.THD_VARLEN: # Dynamic packed token totals: one symbol per ragged group (Q/O and @@ -2337,12 +2339,16 @@ def _fake_bshd(shape, stride, dtype=STORAGE_DTYPE, bpe=CFG.BPE): ) else: if lse_head_major or lse_head_stride: - raise ValueError("lse_head_major / lse_head_stride are THD-only (dense LSE is compact (B, H, Sq))") - fake_lse = cute.runtime.make_fake_compact_tensor( - cutlass.Float32, - (_lse_batch, qh, sq), - stride_order=(2, 1, 0), - assumed_align=16, + raise ValueError("lse_head_major / lse_head_stride are THD-only") + fake_lse = ( + cute.runtime.make_fake_tensor(cutlass.Float32, (_lse_batch, qh, sq), lse_stride, assumed_align=4) + if lse_stride is not None + else cute.runtime.make_fake_compact_tensor( + cutlass.Float32, + (_lse_batch, qh, sq), + stride_order=(2, 1, 0), + assumed_align=16, + ) ) # Sinks tensor always part of the ABI; read only when CFG.HAS_SINK == 1 (compile-time fold). fake_sinks = cute.runtime.make_fake_compact_tensor( diff --git a/python/cudnn/sdpa/fwd/kernels/prefill_d128_fp8_sm100.py b/python/cudnn/sdpa/fwd/kernels/prefill_d128_fp8_sm100.py index 17aa904b1c..25d97b022d 100644 --- a/python/cudnn/sdpa/fwd/kernels/prefill_d128_fp8_sm100.py +++ b/python/cudnn/sdpa/fwd/kernels/prefill_d128_fp8_sm100.py @@ -1910,8 +1910,7 @@ def _correction_warp_group( # TMA-STG put the chunk's O. The pair (O_s, lse_s) is everything # the combine needs. Folds to batch_idx at SPLIT_KV == 1. lse_batch = _partial_batch(batch_idx, split_idx, n_batch) - lse_row = lse_arr[lse_batch, row_head_idx, :] - lse_row[q_row_global] = lse_val + lse_arr[lse_batch, row_head_idx, q_row_global] = lse_val # amax_o = max over valid rows of |o_scaled| (the fp32 pre-cast output). Divided # by scale_o in api to give the pre-quant output amax (cuDNN FP8 ref, in-kernel). @@ -2232,6 +2231,7 @@ def compile( # noqa: A001 has_lse: bool = True, lse_head_major: bool = False, lse_head_stride: int = 0, + lse_stride: Optional[tuple[int, int, int]] = None, d_qk: int = CFG.TILE_K, d_v: int = CFG.TILE_O, ) -> Callable: @@ -2335,11 +2335,15 @@ def compile( # noqa: A001 else: if lse_head_major or lse_head_stride: raise ValueError("lse_head_major / lse_head_stride are THD-only (dense LSE is compact (B, H, Sq))") - fake_lse = cute.runtime.make_fake_compact_tensor( - cutlass.Float32, - (_lse_batch, qh, sq), - stride_order=(2, 1, 0), - assumed_align=16, + fake_lse = ( + cute.runtime.make_fake_tensor(cutlass.Float32, (_lse_batch, qh, sq), lse_stride, assumed_align=4) + if lse_stride is not None and SPLIT_KV == 1 + else cute.runtime.make_fake_compact_tensor( + cutlass.Float32, + (_lse_batch, qh, sq), + stride_order=(2, 1, 0), + assumed_align=16, + ) ) # Always part of the ABI; unread when CFG.HAS_SINK == 0 (compile-time fold). fake_sinks = cute.runtime.make_fake_compact_tensor( diff --git a/python/cudnn/sdpa/fwd/kernels/prefill_d128_fp8_sm107.py b/python/cudnn/sdpa/fwd/kernels/prefill_d128_fp8_sm107.py index d466966162..7ec3207a5c 100644 --- a/python/cudnn/sdpa/fwd/kernels/prefill_d128_fp8_sm107.py +++ b/python/cudnn/sdpa/fwd/kernels/prefill_d128_fp8_sm107.py @@ -2009,8 +2009,7 @@ def _correction_warp_group( if _row_valid: if cutlass.const_expr(lse_tensor is not None): lse_arr = cutlass.make_array_view(lse_tensor) - lse_row = lse_arr[batch_idx, row_head_idx, :] - lse_row[q_row_global] = lse_val + lse_arr[batch_idx, row_head_idx, q_row_global] = lse_val # amax_o = max over valid rows of |o_scaled| (the fp32 pre-cast output). Divided # by scale_o in api to give the pre-quant output amax (cuDNN FP8 ref, in-kernel). @@ -2321,6 +2320,7 @@ def compile( # noqa: A001 has_lse: bool = True, lse_head_major: bool = False, lse_head_stride: int = 0, + lse_stride: Optional[tuple[int, int, int]] = None, d_qk: int = CFG.TILE_K, d_v: int = CFG.TILE_O, ) -> Callable: @@ -2414,11 +2414,15 @@ def compile( # noqa: A001 else: if lse_head_major or lse_head_stride: raise ValueError("lse_head_major / lse_head_stride are THD-only (dense LSE is compact (B, H, Sq))") - fake_lse = cute.runtime.make_fake_compact_tensor( - cutlass.Float32, - (b, qh, sq), - stride_order=(2, 1, 0), - assumed_align=16, + fake_lse = ( + cute.runtime.make_fake_tensor(cutlass.Float32, (b, qh, sq), lse_stride, assumed_align=4) + if lse_stride is not None + else cute.runtime.make_fake_compact_tensor( + cutlass.Float32, + (b, qh, sq), + stride_order=(2, 1, 0), + assumed_align=16, + ) ) # Always part of the ABI; unread when CFG.HAS_SINK == 0 (compile-time fold). fake_sinks = cute.runtime.make_fake_compact_tensor( diff --git a/python/cudnn/sdpa/fwd/kernels/prefill_d128_mxfp8_sm100.py b/python/cudnn/sdpa/fwd/kernels/prefill_d128_mxfp8_sm100.py index f45059f446..b54e5800fb 100644 --- a/python/cudnn/sdpa/fwd/kernels/prefill_d128_mxfp8_sm100.py +++ b/python/cudnn/sdpa/fwd/kernels/prefill_d128_mxfp8_sm100.py @@ -2320,8 +2320,7 @@ def _correction_warp_group( # TMA-STG put the chunk's O. The pair (O_s, lse_s) is everything # the combine needs. Folds to batch_idx at SPLIT_KV == 1. lse_batch = _partial_batch(batch_idx, split_idx, n_batch) - lse_row = lse_arr[lse_batch, head_idx, :] - lse_row[q_row_global] = lse_val + lse_arr[lse_batch, head_idx, q_row_global] = lse_val sO_sub_base = sO[qs].base @@ -2623,6 +2622,7 @@ def compile( # noqa: A001 has_lse: bool = True, lse_head_major: bool = False, lse_head_stride: int = 0, + lse_stride: Optional[tuple[int, int, int]] = None, ) -> Callable: """Compile a kernel with concrete dims; 3 SF tensors layout [B, H, num_seq_tiles, SF_SMEM_SIZE_*]. @@ -2736,11 +2736,15 @@ def compile( # noqa: A001 else: if lse_head_major or lse_head_stride: raise ValueError("lse_head_major / lse_head_stride are THD-only (dense LSE is compact (B, H, Sq))") - fake_lse = cute.runtime.make_fake_compact_tensor( - cutlass.Float32, - (_lse_batch, qh, sq), - stride_order=(2, 1, 0), - assumed_align=16, + fake_lse = ( + cute.runtime.make_fake_tensor(cutlass.Float32, (_lse_batch, qh, sq), lse_stride, assumed_align=4) + if lse_stride is not None and SPLIT_KV == 1 + else cute.runtime.make_fake_compact_tensor( + cutlass.Float32, + (_lse_batch, qh, sq), + stride_order=(2, 1, 0), + assumed_align=16, + ) ) fake_amax_o = cute.runtime.make_fake_compact_tensor( cutlass.Float32, diff --git a/python/cudnn/sdpa/fwd/kernels/prefill_d192_d128_f16_sm100.py b/python/cudnn/sdpa/fwd/kernels/prefill_d192_d128_f16_sm100.py index 87da77e16c..9911e6c146 100644 --- a/python/cudnn/sdpa/fwd/kernels/prefill_d192_d128_f16_sm100.py +++ b/python/cudnn/sdpa/fwd/kernels/prefill_d192_d128_f16_sm100.py @@ -2081,8 +2081,7 @@ def _correction_warp_group( # where TMA-STG put the chunk's O. The pair (O_s, lse_s) is # everything the combine needs. lse_batch = _partial_batch(batch_idx, split_idx, n_batch) - lse_row = lse_arr[lse_batch, row_head_idx, :] - lse_row[q_row_global] = lse_val + lse_arr[lse_batch, row_head_idx, q_row_global] = lse_val sO_sub_base = sO[qs].base @@ -2327,6 +2326,7 @@ def compile( # noqa: A001 k_stride: Optional[tuple] = None, v_stride: Optional[tuple] = None, o_stride: Optional[tuple] = None, + lse_stride: Optional[tuple[int, int, int]] = None, ) -> Callable: """Compile a kernel with ALL dims concrete to pin TMA descriptor strides at compile time. @@ -2356,6 +2356,8 @@ def compile( # noqa: A001 # Each split's LSE is not optional under KV split — it IS the weight # the combine reduces with. Without it the partials cannot be recombined. raise ValueError("split_kv > 1 requires has_lse=True (the per-split LSE drives the combine)") + if lse_stride is not None and (CFG.THD_VARLEN or SPLIT_KV > 1): + raise ValueError("dense LSE strides are not valid for THD or split-KV workspaces") _fake_batch = 1 if CFG.THD_VARLEN else b if CFG.THD_VARLEN: # Dynamic packed token totals: one symbol per ragged group (Q/O and @@ -2422,12 +2424,16 @@ def _fake_bshd(shape, stride, dtype=STORAGE_DTYPE, bpe=CFG.BPE): ) else: if lse_head_major or lse_head_stride: - raise ValueError("lse_head_major / lse_head_stride are THD-only (dense LSE is compact (B, H, Sq))") - fake_lse = cute.runtime.make_fake_compact_tensor( - cutlass.Float32, - (_lse_batch, qh, sq), - stride_order=(2, 1, 0), - assumed_align=16, + raise ValueError("lse_head_major / lse_head_stride are THD-only") + fake_lse = ( + cute.runtime.make_fake_tensor(cutlass.Float32, (_lse_batch, qh, sq), lse_stride, assumed_align=4) + if lse_stride is not None + else cute.runtime.make_fake_compact_tensor( + cutlass.Float32, + (_lse_batch, qh, sq), + stride_order=(2, 1, 0), + assumed_align=16, + ) ) # Sinks tensor always part of the ABI; read only when CFG.HAS_SINK == 1 (compile-time fold). fake_sinks = cute.runtime.make_fake_compact_tensor( diff --git a/python/cudnn/sdpa/fwd/kernels/prefill_d192_d128_fp8_sm100.py b/python/cudnn/sdpa/fwd/kernels/prefill_d192_d128_fp8_sm100.py index d705176bf5..b7396fa131 100644 --- a/python/cudnn/sdpa/fwd/kernels/prefill_d192_d128_fp8_sm100.py +++ b/python/cudnn/sdpa/fwd/kernels/prefill_d192_d128_fp8_sm100.py @@ -2003,8 +2003,7 @@ def _correction_warp_group( if _row_valid: if cutlass.const_expr(lse_tensor is not None): lse_arr = cutlass.make_array_view(lse_tensor) - lse_row = lse_arr[batch_idx, row_head_idx, :] - lse_row[q_row_global] = lse_val + lse_arr[batch_idx, row_head_idx, q_row_global] = lse_val # amax_o is defined over the fp32 pre-cast output. _amax_o_ptr = Pointer(amax_o_tensor.iterator.raw_ptr(), dtype=cutlass.Int32) @@ -2292,6 +2291,7 @@ def compile( # noqa: A001 sq: int = 256, skv: int = 128, has_lse: bool = True, + lse_stride: Optional[tuple[int, int, int]] = None, d_qk: int = CFG.TILE_K, d_v: int = CFG.TILE_O, ) -> Callable: @@ -2337,16 +2337,17 @@ def compile( # noqa: A001 stride_order=(3, 2, 1, 0), assumed_align=16, ) - fake_lse = ( - cute.runtime.make_fake_compact_tensor( + if not has_lse: + fake_lse = None + elif lse_stride is not None: + fake_lse = cute.runtime.make_fake_tensor(cutlass.Float32, (_fake_batch, qh, sq), lse_stride, assumed_align=4) + else: + fake_lse = cute.runtime.make_fake_compact_tensor( cutlass.Float32, (_fake_batch, qh, sq), stride_order=(2, 1, 0), assumed_align=16, ) - if has_lse - else None - ) # Always part of the ABI; unread when CFG.HAS_SINK == 0 (compile-time fold). fake_sinks = cute.runtime.make_fake_compact_tensor( cutlass.Float32, diff --git a/python/cudnn/sdpa/fwd/kernels/prefill_d192_d128_mxfp8_sm100.py b/python/cudnn/sdpa/fwd/kernels/prefill_d192_d128_mxfp8_sm100.py index 1015e6a7da..28710f45c4 100644 --- a/python/cudnn/sdpa/fwd/kernels/prefill_d192_d128_mxfp8_sm100.py +++ b/python/cudnn/sdpa/fwd/kernels/prefill_d192_d128_mxfp8_sm100.py @@ -2449,8 +2449,7 @@ def _correction_warp_group( if cutlass.const_expr(lse_tensor is not None): if _row_valid: lse_arr = cutlass.make_array_view(lse_tensor) - lse_row = lse_arr[batch_idx, head_idx, :] - lse_row[q_row_global] = lse_val + lse_arr[batch_idx, head_idx, q_row_global] = lse_val sO_sub_base = sO[qs].base @@ -2702,6 +2701,7 @@ def compile( # noqa: A001 sq: int = 256, skv: int = 128, has_lse: bool = True, + lse_stride: Optional[tuple[int, int, int]] = None, ) -> Callable: """Compile a kernel with concrete dims; 3 SF tensors layout [B, H, num_seq_tiles, SF_SMEM_SIZE_*]. @@ -2764,11 +2764,15 @@ def compile( # noqa: A001 # is compiled out entirely — no dummy buffer exists at any level. fake_lse = None else: - fake_lse = cute.runtime.make_fake_compact_tensor( - cutlass.Float32, - (b, qh, sq), - stride_order=(2, 1, 0), - assumed_align=16, + fake_lse = ( + cute.runtime.make_fake_tensor(cutlass.Float32, (b, qh, sq), lse_stride, assumed_align=4) + if lse_stride is not None + else cute.runtime.make_fake_compact_tensor( + cutlass.Float32, + (b, qh, sq), + stride_order=(2, 1, 0), + assumed_align=16, + ) ) fake_amax_o = cute.runtime.make_fake_compact_tensor( cutlass.Float32, diff --git a/python/cudnn/sdpa/fwd/kernels/prefill_d256_f16_sm100.py b/python/cudnn/sdpa/fwd/kernels/prefill_d256_f16_sm100.py index b5f17f6f14..1c31d0f49c 100644 --- a/python/cudnn/sdpa/fwd/kernels/prefill_d256_f16_sm100.py +++ b/python/cudnn/sdpa/fwd/kernels/prefill_d256_f16_sm100.py @@ -1606,8 +1606,7 @@ def _correction_warp_group( # TMA-STG put the chunk's O. The pair (O_s, lse_s) is everything # the combine needs. lse_batch = _partial_batch(batch_idx, split_idx, n_batch) - lse_row = lse_arr[lse_batch, row_head_idx, :] - lse_row[q_row_global] = lse_val + lse_arr[lse_batch, row_head_idx, q_row_global] = lse_val parity_last_rt = cutlass.Int32(0) if bounds.right > bounds.left: @@ -1831,6 +1830,7 @@ def compile( # noqa: A001 k_stride: Optional[tuple] = None, v_stride: Optional[tuple] = None, o_stride: Optional[tuple] = None, + lse_stride: Optional[tuple[int, int, int]] = None, ) -> Callable: """ENVELOPE: ``d_qk`` / ``d_v`` are the ACTUAL head dims (defaults = full TILE_K / TILE_O). TMA descriptors carry these extents while the tile box @@ -1852,6 +1852,8 @@ def compile( # noqa: A001 # Each split's LSE is not optional under KV split — it IS the weight # the combine reduces with. Without it the partials cannot be recombined. raise ValueError("split_kv > 1 requires has_lse=True (the per-split LSE drives the combine)") + if lse_stride is not None and (CFG.THD_VARLEN or SPLIT_KV > 1): + raise ValueError("dense LSE strides are not valid for THD or split-KV workspaces") _fake_batch = 1 if CFG.THD_VARLEN else b if CFG.THD_VARLEN: # Dynamic packed token totals: one symbol per ragged group (Q/O and @@ -1918,12 +1920,16 @@ def _fake_bshd(shape, stride, dtype=STORAGE_DTYPE, bpe=CFG.BPE): ) else: if lse_head_major or lse_head_stride: - raise ValueError("lse_head_major / lse_head_stride are THD-only (dense LSE is compact (B, H, Sq))") - fake_lse = cute.runtime.make_fake_compact_tensor( - cutlass.Float32, - (_lse_batch, qh, sq), - stride_order=(2, 1, 0), - assumed_align=16, + raise ValueError("lse_head_major / lse_head_stride are THD-only") + fake_lse = ( + cute.runtime.make_fake_tensor(cutlass.Float32, (_lse_batch, qh, sq), lse_stride, assumed_align=4) + if lse_stride is not None + else cute.runtime.make_fake_compact_tensor( + cutlass.Float32, + (_lse_batch, qh, sq), + stride_order=(2, 1, 0), + assumed_align=16, + ) ) fake_sinks = cute.runtime.make_fake_compact_tensor( cutlass.Float32, diff --git a/python/cudnn/sdpa/fwd/kernels/prefill_d256_f16_sm80.py b/python/cudnn/sdpa/fwd/kernels/prefill_d256_f16_sm80.py index 9cce734225..3b0db0faa6 100644 --- a/python/cudnn/sdpa/fwd/kernels/prefill_d256_f16_sm80.py +++ b/python/cudnn/sdpa/fwd/kernels/prefill_d256_f16_sm80.py @@ -440,26 +440,28 @@ def _sdpa_kernel( # Per-tile GMEM element offsets to (batch, q_row_base, head, 0). q_row_base = q_tile_idx * tile_m - SQ_i64 = cutlass.Int64(SQ) SKV_i64 = cutlass.Int64(SKV) - H_i64 = cutlass.Int64(H) H_kv_i64 = cutlass.Int64(H_kv) # Seq origin (in rows) absorbs both the dense batch*S term and the THD # cu_*[b] packed offset — the row-stride multiply is identical either way. q_seq_abs64 = q_seq_origin + cutlass.Int64(q_row_base) Q_BASE = q_seq_abs64 * Q_ROW_STRIDE_E.to(cutlass.Int64) + cutlass.Int64(head_idx) * d_runtime64 O_BASE = q_seq_abs64 * O_ROW_STRIDE_E.to(cutlass.Int64) + cutlass.Int64(head_idx) * d_v64 - # LSE: dense [B, H_q, SQ]; THD packed [1, H_q, T] — element-contiguous along - # the seq axis. THD uses the packed seq position (cu_q[b] + q_row_base). + # LSE: dense [B, H_q, SQ]; THD packed [1, H_q, T]. Dense strides are + # compile-time layout metadata; THD remains compact. # None-specialized: no Stats output ⇒ view/base/pointer are compiled out # together with the epilogue store below. if cutlass.const_expr(LSE is not None): LSE_view = cutlass.make_array_view(LSE) + LSE_S_STRIDE_E = cutlass.Int64(LSE.stride[2]) if cutlass.const_expr(THD_VARLEN): - T_lse_i64 = cutlass.Int64(LSE.shape[2]) - LSE_BASE = cutlass.Int64(head_idx) * T_lse_i64 + q_seq_origin + cutlass.Int64(q_row_base) + LSE_BASE = cutlass.Int64(head_idx) * cutlass.Int64(LSE.stride[1]) + (q_seq_origin + cutlass.Int64(q_row_base)) * LSE_S_STRIDE_E else: - LSE_BASE = cutlass.Int64(batch_idx) * H_i64 * SQ_i64 + cutlass.Int64(head_idx) * SQ_i64 + cutlass.Int64(q_row_base) + LSE_BASE = ( + cutlass.Int64(batch_idx) * cutlass.Int64(LSE.stride[0]) + + cutlass.Int64(head_idx) * cutlass.Int64(LSE.stride[1]) + + cutlass.Int64(q_row_base) * LSE_S_STRIDE_E + ) lse_gmem = LSE_view.data_ptr() + LSE_BASE # K/V offsets parameterised on kv_row_base (variable in mainloop). Uses @@ -1413,8 +1415,8 @@ def _mask_term(col, q_abs, q_mw): trim_bot = q_row_base_i32 + block_row_bot lse_top = cutlass.Float32(arith.select((trim_top < eff_sq).ir_value(), lse_top.ir_value(), _ninf.ir_value())) lse_bot = cutlass.Float32(arith.select((trim_bot < eff_sq).ir_value(), lse_bot.ir_value(), _ninf.ir_value())) - lse_top_ptr = lse_gmem + cutlass.Int64(block_row_top) - lse_bot_ptr = lse_gmem + cutlass.Int64(block_row_bot) + lse_top_ptr = lse_gmem + cutlass.Int64(block_row_top) * LSE_S_STRIDE_E + lse_bot_ptr = lse_gmem + cutlass.Int64(block_row_bot) * LSE_S_STRIDE_E # Row predication for OOB Q-rows when ~is_even_mn. All 4 lanes of a # threadquad share the same row → predicate is uniform within the @@ -1691,6 +1693,7 @@ def compile( # noqa: A001 — the template contract's entry point (matches the swa_window: int = 0, rope_max_s: int = 0, n_batch_logical: int = 0, + lse_stride: Optional[tuple[int, int, int]] = None, ): """Compile (or fetch) this template specialization for one shape. @@ -1763,11 +1766,15 @@ def compile( # noqa: A001 — the template contract's entry point (matches the # LSE: dense [B, H, SQ]; THD packed [1, H, T] (shares the Q/O token # symbol, which is exactly what the kernel's LSE.shape[2] read needs). if p.has_lse: - fake_lse = cute.runtime.make_fake_compact_tensor( - cutlass.Float32, - (_b, h, _sq), - stride_order=(2, 1, 0), - assumed_align=16, + fake_lse = ( + cute.runtime.make_fake_tensor(cutlass.Float32, (_b, h, _sq), lse_stride, assumed_align=4) + if lse_stride is not None and not p.thd_varlen + else cute.runtime.make_fake_compact_tensor( + cutlass.Float32, + (_b, h, _sq), + stride_order=(2, 1, 0), + assumed_align=16, + ) ) else: fake_lse = None diff --git a/python/cudnn/sdpa/fwd/kernels/prefill_d512_f16_sm100.py b/python/cudnn/sdpa/fwd/kernels/prefill_d512_f16_sm100.py index 73b2e718ae..19d75d609a 100644 --- a/python/cudnn/sdpa/fwd/kernels/prefill_d512_f16_sm100.py +++ b/python/cudnn/sdpa/fwd/kernels/prefill_d512_f16_sm100.py @@ -1219,8 +1219,7 @@ def _compute_warp_group( if q_row_global < seqlen_q: lse_arr = cutlass.make_array_view(lse_tensor) lse_batch = _partial_batch(batch_idx, split_idx, n_batch) - lse_row = lse_arr[lse_batch, row_head_idx, :] - lse_row[q_row_global] = lse + lse_arr[lse_batch, row_head_idx, q_row_global] = lse wait(sched.mb_scheduler.subview(sched_state.idx), sched_state.phase) nxt_q = cute.arch.make_warp_uniform(sched.tile_id_smem.subview(sched_state.idx * cutlass.Int32(8) + cutlass.Int32(0)).load()) @@ -2034,6 +2033,7 @@ def compile( # noqa: A001 k_stride: Optional[tuple] = None, v_stride: Optional[tuple] = None, o_stride: Optional[tuple] = None, + lse_stride: Optional[tuple[int, int, int]] = None, ) -> Callable: """ENVELOPE: ``d_qk`` / ``d_v`` are the ACTUAL head dims (defaults = full TILE_K / TILE_O). TMA descriptors carry these extents while the tile box @@ -2055,6 +2055,8 @@ def compile( # noqa: A001 raise ValueError(f"d512 envelope: d_qk*BPE and d_v*BPE must be 16-byte multiples (TMA global-stride rule); got ({d_qk}, {d_v}) at BPE={CFG.BPE}") if SPLIT_KV > 1 and not has_lse: raise ValueError("split_kv > 1 requires has_lse=True (the per-split LSE drives the combine)") + if lse_stride is not None and (CFG.THD_VARLEN or SPLIT_KV > 1): + raise ValueError("dense LSE strides are not valid for THD or split-KV workspaces") _fake_batch = 1 if CFG.THD_VARLEN else b if CFG.THD_VARLEN: # Dynamic packed token totals: one symbol per ragged group (Q/O and @@ -2119,12 +2121,16 @@ def _fake_bshd(shape, stride, dtype=STORAGE_DTYPE, bpe=CFG.BPE): ) else: if lse_head_major or lse_head_stride: - raise ValueError("lse_head_major / lse_head_stride are THD-only (dense LSE is compact (B, H, Sq))") - fake_lse = cute.runtime.make_fake_compact_tensor( - cutlass.Float32, - (_lse_batch, qh, sq), - stride_order=(2, 1, 0), - assumed_align=16, + raise ValueError("lse_head_major / lse_head_stride are THD-only") + fake_lse = ( + cute.runtime.make_fake_tensor(cutlass.Float32, (_lse_batch, qh, sq), lse_stride, assumed_align=4) + if lse_stride is not None + else cute.runtime.make_fake_compact_tensor( + cutlass.Float32, + (_lse_batch, qh, sq), + stride_order=(2, 1, 0), + assumed_align=16, + ) ) fake_sinks = cute.runtime.make_fake_compact_tensor( cutlass.Float32, diff --git a/python/cudnn/sdpa/fwd/kernels/prefill_f16_sm120.py b/python/cudnn/sdpa/fwd/kernels/prefill_f16_sm120.py index 52897950bd..1abc919454 100644 --- a/python/cudnn/sdpa/fwd/kernels/prefill_f16_sm120.py +++ b/python/cudnn/sdpa/fwd/kernels/prefill_f16_sm120.py @@ -1299,8 +1299,7 @@ def kernel( if lse_q_idx >= seqlen_q: lse_out = -cutlass.Float32.inf if lse_q_idx < q.shape[1]: - lse_row = lse_arr[o_batch_idx, _lse_head, :] - lse_row[lse_q_idx] = lse_out + lse_arr[o_batch_idx, _lse_head, lse_q_idx] = lse_out prims.barrier_cta_sync(self.bar_compute_sync, thread_count=self.threads_compute) @@ -1480,8 +1479,6 @@ def _static_neq(a, b): # batch mode B*SPLIT_KV (same as O). if cutlass.const_expr(lse.shape != (q.shape[0] * self.split_kv, q.shape[2], q.shape[1])): raise ValueError("LSE must have shape (B * split_kv, H, Sq)") - if cutlass.const_expr(lse.stride != (q.shape[2] * q.shape[1], q.shape[1], 1)): - raise ValueError("LSE must be compact row-major") if cutlass.const_expr(self.has_sink != (sinks is not None)): raise ValueError("sinks must be provided exactly when the kernel is configured with has_sink") if cutlass.const_expr(sinks is not None and sinks.shape != (q.shape[2],)): @@ -1605,6 +1602,7 @@ def compile( # noqa: A001 k_stride: Optional[tuple[int, int, int, int]] = None, v_stride: Optional[tuple[int, int, int, int]] = None, o_stride: Optional[tuple[int, int, int, int]] = None, + lse_stride: Optional[tuple[int, int, int]] = None, ) -> Callable: """Compile and cache one architecture-specific compact BSHD shape. @@ -1623,7 +1621,9 @@ def compile( # noqa: A001 ``has_lse=False`` compiles the LSE store out (the kernel specializes on a ``None`` LSE argument) — callers that don't want stats pass no LSE buffer - at all instead of a dummy. THD LSE is token-major ``(T, H)`` by default; + at all instead of a dummy. Dense ``lse_stride`` carries the caller's + declared ``(B, H, Sq)`` element strides into the compiled tensor. THD LSE + is token-major ``(T, H)`` by default; ``lse_head_major=True`` switches to head-major ``(H, lse_head_stride)`` (FlashAttention's ``softmax_lse`` layout), where ``lse_head_stride`` is the caller-declared head-row stride (``>= T``, a shape — part of the cache key). @@ -1653,6 +1653,8 @@ def compile( # noqa: A001 ) if PARAMS.split_kv > 1 and not has_lse: raise ValueError("SM120 SDPA: split_kv > 1 requires an LSE output (the per-split LSE drives the combine)") + if lse_stride is not None and (PARAMS.thd_varlen or PARAMS.split_kv > 1): + raise ValueError("dense LSE strides are not valid for THD or split-KV workspaces") fake_batch = 1 if PARAMS.thd_varlen else b if PARAMS.thd_varlen: # Dynamic packed token totals: one symbol per ragged group (Q/O and @@ -1682,16 +1684,21 @@ def _fake_bshd(shape, stride): fake_lse_shape = (qh, lse_head_stride) if lse_head_major else (sq, qh) else: fake_lse_shape = (lse_fake_batch, qh, sq) - fake_lse = ( - cute.runtime.make_fake_compact_tensor( - cutlass.Float32, - fake_lse_shape, - stride_order=(1, 0) if PARAMS.thd_varlen else (2, 1, 0), - assumed_align=4, + if not has_lse: + # No Stats output: the LSE argument is None-specialized and the store + # is compiled out entirely — no dummy buffer exists at any level. + fake_lse = None + else: + fake_lse = ( + cute.runtime.make_fake_tensor(cutlass.Float32, fake_lse_shape, lse_stride, assumed_align=4) + if lse_stride is not None + else cute.runtime.make_fake_compact_tensor( + cutlass.Float32, + fake_lse_shape, + stride_order=(1, 0) if PARAMS.thd_varlen else (2, 1, 0), + assumed_align=4, + ) ) - if has_lse - else None - ) fake_sinks = ( cute.runtime.make_fake_compact_tensor( cutlass.Float32, diff --git a/python/cudnn/sdpa/fwd/kernels/prefill_f16_sm80.py b/python/cudnn/sdpa/fwd/kernels/prefill_f16_sm80.py index 8275aa2498..9af33a06e8 100644 --- a/python/cudnn/sdpa/fwd/kernels/prefill_f16_sm80.py +++ b/python/cudnn/sdpa/fwd/kernels/prefill_f16_sm80.py @@ -468,26 +468,28 @@ def _sdpa_kernel( # Per-tile GMEM element offsets to (batch, q_row_base, head, 0). q_row_base = q_tile_idx * tile_m - SQ_i64 = cutlass.Int64(SQ) SKV_i64 = cutlass.Int64(SKV) - H_i64 = cutlass.Int64(H) H_kv_i64 = cutlass.Int64(H_kv) # Seq origin (in rows) absorbs both the dense batch*S term and the THD # cu_*[b] packed offset — the row-stride multiply is identical either way. q_seq_abs64 = q_seq_origin + cutlass.Int64(q_row_base) Q_BASE = q_seq_abs64 * Q_ROW_STRIDE_E.to(cutlass.Int64) + cutlass.Int64(head_idx) * d_runtime64 O_BASE = q_seq_abs64 * O_ROW_STRIDE_E.to(cutlass.Int64) + cutlass.Int64(head_idx) * d_v64 - # LSE: dense [B, H_q, SQ]; THD packed [1, H_q, T] — element-contiguous along - # the seq axis. THD uses the packed seq position (cu_q[b] + q_row_base). + # LSE: dense [B, H_q, SQ]; THD packed [1, H_q, T]. Dense strides are + # compile-time layout metadata; THD remains compact. # None-specialized: no Stats output ⇒ view/base/pointer are compiled out # together with the epilogue store below. if cutlass.const_expr(LSE is not None): LSE_view = cutlass.make_array_view(LSE) + LSE_S_STRIDE_E = cutlass.Int64(LSE.stride[2]) if cutlass.const_expr(THD_VARLEN): - T_lse_i64 = cutlass.Int64(LSE.shape[2]) - LSE_BASE = cutlass.Int64(head_idx) * T_lse_i64 + q_seq_origin + cutlass.Int64(q_row_base) + LSE_BASE = cutlass.Int64(head_idx) * cutlass.Int64(LSE.stride[1]) + (q_seq_origin + cutlass.Int64(q_row_base)) * LSE_S_STRIDE_E else: - LSE_BASE = cutlass.Int64(batch_idx) * H_i64 * SQ_i64 + cutlass.Int64(head_idx) * SQ_i64 + cutlass.Int64(q_row_base) + LSE_BASE = ( + cutlass.Int64(batch_idx) * cutlass.Int64(LSE.stride[0]) + + cutlass.Int64(head_idx) * cutlass.Int64(LSE.stride[1]) + + cutlass.Int64(q_row_base) * LSE_S_STRIDE_E + ) lse_gmem = LSE_view.data_ptr() + LSE_BASE # K/V offsets parameterised on kv_row_base (variable in mainloop). Uses @@ -1412,8 +1414,8 @@ def _mask_term(col, q_abs, q_mw): trim_bot = q_row_base_i32 + block_row_bot lse_top = cutlass.Float32(arith.select((trim_top < eff_sq).ir_value(), lse_top.ir_value(), _ninf.ir_value())) lse_bot = cutlass.Float32(arith.select((trim_bot < eff_sq).ir_value(), lse_bot.ir_value(), _ninf.ir_value())) - lse_top_ptr = lse_gmem + cutlass.Int64(block_row_top) - lse_bot_ptr = lse_gmem + cutlass.Int64(block_row_bot) + lse_top_ptr = lse_gmem + cutlass.Int64(block_row_top) * LSE_S_STRIDE_E + lse_bot_ptr = lse_gmem + cutlass.Int64(block_row_bot) * LSE_S_STRIDE_E # Row predication for OOB Q-rows when ~is_even_mn. All 4 lanes of a # threadquad share the same row → predicate is uniform within the @@ -1690,6 +1692,7 @@ def compile( # noqa: A001 — the template contract's entry point (matches the swa_window: int = 0, rope_max_s: int = 0, n_batch_logical: int = 0, + lse_stride: Optional[tuple[int, int, int]] = None, ): """Compile (or fetch) this template specialization for one shape. @@ -1762,11 +1765,15 @@ def compile( # noqa: A001 — the template contract's entry point (matches the # LSE: dense [B, H, SQ]; THD packed [1, H, T] (shares the Q/O token # symbol, which is exactly what the kernel's LSE.shape[2] read needs). if p.has_lse: - fake_lse = cute.runtime.make_fake_compact_tensor( - cutlass.Float32, - (_b, h, _sq), - stride_order=(2, 1, 0), - assumed_align=16, + fake_lse = ( + cute.runtime.make_fake_tensor(cutlass.Float32, (_b, h, _sq), lse_stride, assumed_align=4) + if lse_stride is not None and not p.thd_varlen + else cute.runtime.make_fake_compact_tensor( + cutlass.Float32, + (_b, h, _sq), + stride_order=(2, 1, 0), + assumed_align=16, + ) ) else: fake_lse = None diff --git a/python/cudnn/sdpa/fwd/kernels/prefill_fp8_sm120.py b/python/cudnn/sdpa/fwd/kernels/prefill_fp8_sm120.py index a23ce79eb8..6bf5229b88 100644 --- a/python/cudnn/sdpa/fwd/kernels/prefill_fp8_sm120.py +++ b/python/cudnn/sdpa/fwd/kernels/prefill_fp8_sm120.py @@ -1446,8 +1446,7 @@ def kernel( if lse_q_idx >= seqlen_q: lse_out = -cutlass.Float32.inf if lse_q_idx < q.shape[1]: - lse_row = lse_arr[batch_idx, _lse_head, :] - lse_row[lse_q_idx] = lse_out + lse_arr[batch_idx, _lse_head, lse_q_idx] = lse_out prims.barrier_cta_sync(self.bar_compute_sync, thread_count=self.threads_compute) @@ -1684,8 +1683,6 @@ def _static_neq(a, b): else: if cutlass.const_expr(lse.shape != (q.shape[0], q.shape[2], q.shape[1])): raise ValueError("LSE must have shape (B, H, Sq)") - if cutlass.const_expr(lse.stride != (q.shape[2] * q.shape[1], q.shape[1], 1)): - raise ValueError("LSE must be compact row-major") if cutlass.const_expr(self.has_sink != (sinks is not None)): raise ValueError("sinks must be provided exactly when the kernel is configured with has_sink") if cutlass.const_expr(sinks is not None and sinks.shape != (q.shape[2],)): @@ -1797,6 +1794,7 @@ def compile( # noqa: A001 has_lse: bool = True, lse_head_major: bool = False, lse_head_stride: int = 0, + lse_stride: Optional[tuple[int, int, int]] = None, ) -> Callable: """Compile and cache one architecture-specific compact BSHD shape. @@ -1813,7 +1811,8 @@ def compile( # noqa: A001 ``has_lse=False`` compiles the LSE store out (the kernel specializes on a ``None`` LSE argument) — callers that don't want stats pass no LSE buffer - at all instead of a dummy. + at all instead of a dummy. Dense ``lse_stride`` carries the caller's + declared ``(B, H, Sq)`` element strides into the compiled tensor. THD LSE is token-major ``(T, H)`` by default; ``lse_head_major=True`` switches to head-major ``(H, lse_head_stride)`` (FlashAttention's @@ -1842,6 +1841,8 @@ def compile( # noqa: A001 pack_gqa=PARAMS.pack_gqa, qh_per_kh=qh // kh, ) + if has_lse and lse_stride is not None and PARAMS.thd_varlen: + raise ValueError("dense LSE strides are not valid for THD") fake_batch = 1 if PARAMS.thd_varlen else b if PARAMS.thd_varlen: # Dynamic packed token totals: one symbol per ragged group (Q/O share @@ -1873,16 +1874,22 @@ def compile( # noqa: A001 stride_order=(3, 2, 1, 0), assumed_align=16, ) - fake_lse = ( - cute.runtime.make_fake_compact_tensor( - cutlass.Float32, - (((qh, lse_head_stride) if lse_head_major else (sq, qh)) if PARAMS.thd_varlen else (fake_batch, qh, sq)), - stride_order=(1, 0) if PARAMS.thd_varlen else (2, 1, 0), - assumed_align=4, + fake_lse_shape = ((qh, lse_head_stride) if lse_head_major else (sq, qh)) if PARAMS.thd_varlen else (fake_batch, qh, sq) + if not has_lse: + # No Stats output: the LSE argument is None-specialized and the store + # is compiled out entirely — no dummy buffer exists at any level. + fake_lse = None + else: + fake_lse = ( + cute.runtime.make_fake_tensor(cutlass.Float32, fake_lse_shape, lse_stride, assumed_align=4) + if lse_stride is not None + else cute.runtime.make_fake_compact_tensor( + cutlass.Float32, + fake_lse_shape, + stride_order=(1, 0) if PARAMS.thd_varlen else (2, 1, 0), + assumed_align=4, + ) ) - if has_lse - else None - ) fake_sinks = ( cute.runtime.make_fake_compact_tensor( cutlass.Float32, diff --git a/python/cudnn/sdpa/fwd/kernels/split_combine_sm100.py b/python/cudnn/sdpa/fwd/kernels/split_combine_sm100.py index 48aa41800e..5e20826411 100644 --- a/python/cudnn/sdpa/fwd/kernels/split_combine_sm100.py +++ b/python/cudnn/sdpa/fwd/kernels/split_combine_sm100.py @@ -125,8 +125,7 @@ def _combine_kernel( lo = cutlass.make_array_view(lse_out) lse_val = m_safe + cute.math.log(cute.math.max(den, cutlass.Float32(1e-30)), fastmath=True) lse_val = cutlass.Float32(arith.select(all_dead.ir_value(), cutlass.Float32(NEG_INF).ir_value(), lse_val.ir_value())) - lse_row_out = lo[batch, head, :] - lse_row_out[q_row] = lse_val + lo[batch, head, q_row] = lse_val @cute.jit @@ -166,6 +165,7 @@ def compile( # noqa: A001 splits: int, dtype_o: str = "f16", has_lse: bool = False, + lse_stride: Optional[tuple[int, int, int]] = None, has_amax: bool = False, ) -> Callable: """Compile the combine pass for one concrete (B, H, S_q, d_v, splits) shape. @@ -175,14 +175,24 @@ def compile( # noqa: A001 — the pass is bandwidth-bound, so unrolling it buys nothing. ``has_lse`` controls whether the recombined LSE is written at all; with ``False`` the store is None-specialized out of the traced code. ``has_amax`` does the - same for the FP8-family amax of the recombined O. + same for the FP8-family amax of the recombined O. ``lse_stride`` describes + the caller-visible LSE output; the per-split LSE input workspace remains + compact regardless of that final layout. """ elem = {"f16": cutlass.Float16, "bf16": cutlass.BFloat16}[dtype_o] fake_o_partial = cute.runtime.make_fake_compact_tensor(elem, (splits * b, sq, h, d_v), stride_order=(3, 2, 1, 0), assumed_align=16) fake_lse_partial = cute.runtime.make_fake_compact_tensor(cutlass.Float32, (splits * b, h, sq), stride_order=(2, 1, 0), assumed_align=16) fake_o_out = cute.runtime.make_fake_compact_tensor(elem, (b, sq, h, d_v), stride_order=(3, 2, 1, 0), assumed_align=16) - fake_lse_out = cute.runtime.make_fake_compact_tensor(cutlass.Float32, (b, h, sq), stride_order=(2, 1, 0), assumed_align=16) if has_lse else None + fake_lse_out = ( + ( + cute.runtime.make_fake_tensor(cutlass.Float32, (b, h, sq), lse_stride, assumed_align=4) + if lse_stride is not None + else cute.runtime.make_fake_compact_tensor(cutlass.Float32, (b, h, sq), stride_order=(2, 1, 0), assumed_align=16) + ) + if has_lse + else None + ) fake_amax_o = cute.runtime.make_fake_compact_tensor(cutlass.Float32, (1,), stride_order=(0,), assumed_align=4) if has_amax else None return cute.compile( diff --git a/test/python/sdpa/frost/frost_test_utils.py b/test/python/sdpa/frost/frost_test_utils.py index 14e24f079a..46b886f4b4 100644 --- a/test/python/sdpa/frost/frost_test_utils.py +++ b/test/python/sdpa/frost/frost_test_utils.py @@ -111,3 +111,17 @@ def _wanted(i): def offers_engine(graph, name) -> bool: """Whether any ranked entry is a plan for engine ``name``.""" return any(_is_plan_for(graph.get_plan_name_at_index(i), name) for i in range(len(graph.plans))) + + +def make_dense_stats(batch: int, heads: int, sequence: int, layout: str): + """Allocate dense Stats in compact or permuted-and-gapped storage.""" + import torch + + if layout == "contiguous": + return torch.empty(batch, heads, sequence, 1, dtype=torch.float32, device="cuda") + if layout == "strided": + storage = torch.empty(sequence + 7, heads + 2, batch, dtype=torch.float32, device="cuda") + stats = storage.permute(2, 1, 0)[:, :heads, :sequence].unsqueeze(-1) + assert not stats.is_contiguous() + return stats + raise ValueError(f"unknown dense Stats layout {layout!r}") diff --git a/test/python/sdpa/frost/test_sdpa_fwd_dsl_sm100.py b/test/python/sdpa/frost/test_sdpa_fwd_dsl_sm100.py index 1bb22d9d92..28d868d04d 100644 --- a/test/python/sdpa/frost/test_sdpa_fwd_dsl_sm100.py +++ b/test/python/sdpa/frost/test_sdpa_fwd_dsl_sm100.py @@ -12,7 +12,7 @@ from test_utils import torch_fork_set_rng from cudnn.sdpa.fwd.engines import engine_name -from frost_test_utils import requires_pre_rubin_blackwell, requires_dsl, _dsl_installed +from frost_test_utils import make_dense_stats, requires_pre_rubin_blackwell, requires_dsl, _dsl_installed from frost_test_utils import select_engine as _select_engine # noqa: F401 @@ -169,7 +169,21 @@ def _bhsd(b, h, s, d, dtype, device="cuda"): return torch.randn(b, s, h, d, device=device, dtype=dtype).transpose(1, 2) -def _run_dsl_graph(q_gpu, k_gpu, v_gpu, *, scale, dtype, sdpa_kwargs, seq_len_kv=None, seq_len_q=None, sink=None, pack_gqa=None, return_stats=False): +def _run_dsl_graph( + q_gpu, + k_gpu, + v_gpu, + *, + scale, + dtype, + sdpa_kwargs, + seq_len_kv=None, + seq_len_q=None, + sink=None, + pack_gqa=None, + return_stats=False, + stats_layout="contiguous", +): """Build the graph, opt into the matching FROST DSL engine, execute, return O (BHSD).""" import cudnn @@ -203,10 +217,9 @@ def _run_dsl_graph(q_gpu, k_gpu, v_gpu, *, scale, dtype, sdpa_kwargs, seq_len_kv o.set_output(True).set_dim(o_gpu.shape).set_stride(o_gpu.stride()) stats_gpu = None if return_stats: - stats_gpu = torch.empty(b, h_q, s_q, dtype=torch.float32, device="cuda") - stats.set_output(True).set_data_type(cudnn.data_type.FLOAT) - stats.set_dim((b, h_q, s_q, 1)).set_stride((h_q * s_q, s_q, 1, 1)) - vp[stats] = stats_gpu + assert stats is not None + stats_gpu = make_dense_stats(b, h_q, s_q, stats_layout) + stats.set_output(True).set_dim(stats_gpu.shape).set_stride(stats_gpu.stride()).set_data_type(cudnn.data_type.FLOAT) g.validate() g.build_operation_graph() @@ -215,6 +228,8 @@ def _run_dsl_graph(q_gpu, k_gpu, v_gpu, *, scale, dtype, sdpa_kwargs, seq_len_kv g.check_support() g.build_plans() vp[o] = o_gpu + if stats_gpu is not None: + vp[stats] = stats_gpu g.execute(vp, torch.empty(max(g.get_workspace_size(), 1), device="cuda", dtype=torch.uint8)) torch.cuda.synchronize() if return_stats: @@ -222,6 +237,42 @@ def _run_dsl_graph(q_gpu, k_gpu, v_gpu, *, scale, dtype, sdpa_kwargs, seq_len_kv return o_gpu +def _check_dsl_sm100_strided_stats(d_qk, d_v): + _require_dsl() + if torch.cuda.get_device_capability() == (10, 7): + pytest.skip("SM107 serves only the per-tensor FP8 d128 forward path") + b, h, s = 2, 4, 128 + dtype = torch.float16 + scale = 1.0 / math.sqrt(d_qk) + q = _bhsd(b, h, s, d_qk, dtype) + k = _bhsd(b, h, s, d_qk, dtype) + v = _bhsd(b, h, s, d_v, dtype) + + _, contiguous_stats = _run_dsl_graph(q, k, v, scale=scale, dtype=dtype, sdpa_kwargs=dict(use_causal_mask=True), return_stats=True) + o, strided_stats = _run_dsl_graph( + q, k, v, scale=scale, dtype=dtype, sdpa_kwargs=dict(use_causal_mask=True), return_stats=True, stats_layout="strided" + ) + o_ref, stats_ref = _ref_sdpa_full(q, k, v, scale=scale, is_causal=True, return_stats=True) + torch.testing.assert_close(o, o_ref, atol=5e-2, rtol=3e-2) + torch.testing.assert_close(strided_stats, contiguous_stats, atol=0, rtol=0) + torch.testing.assert_close(strided_stats.squeeze(-1), stats_ref, atol=5e-2, rtol=3e-2) + + +@pytest.mark.L0 +@torch_fork_set_rng(seed=59) +def test_dsl_sm100_strided_stats(): + """The SM100 half L0 flavor writes permuted, gapped LSE directly.""" + _check_dsl_sm100_strided_stats(128, 128) + + +@pytest.mark.L1 +@pytest.mark.parametrize(("d_qk", "d_v"), [(192, 128), (256, 256), (512, 512)], ids=["d192_d128", "d256", "d512"]) +@torch_fork_set_rng(seed=59) +def test_dsl_sm100_strided_stats_other_flavors(d_qk, d_v): + """The remaining SM100 half flavors preserve dense Stats strides.""" + _check_dsl_sm100_strided_stats(d_qk, d_v) + + @pytest.mark.L0 @pytest.mark.parametrize("dtype", _DTYPES, ids=_DTYPE_IDS) @pytest.mark.parametrize("is_causal", [False, True], ids=["dense", "causal"]) diff --git a/test/python/sdpa/frost/test_sdpa_fwd_dsl_sm120.py b/test/python/sdpa/frost/test_sdpa_fwd_dsl_sm120.py index c5241fdaf7..9669af6750 100644 --- a/test/python/sdpa/frost/test_sdpa_fwd_dsl_sm120.py +++ b/test/python/sdpa/frost/test_sdpa_fwd_dsl_sm120.py @@ -11,7 +11,7 @@ import torch from test_utils import torch_fork_set_rng -from frost_test_utils import requires_blackwell_geforce, requires_dsl, _dsl_installed +from frost_test_utils import make_dense_stats, requires_blackwell_geforce, requires_dsl, _dsl_installed def _is_sm120() -> bool: @@ -169,6 +169,7 @@ def _run_case( with_sink: bool = False, check_stats: bool = False, pack_gqa: bool | None = None, + stats_layout: str = "contiguous", ) -> None: q = _bhsd(batch, h_q, s_q, head_dim, dtype) k = _bhsd(batch, h_kv, s_kv, head_dim, dtype) @@ -193,6 +194,7 @@ def _run_case( pack_gqa=pack_gqa, scale=scale, return_stats=check_stats, + stats_layout=stats_layout, **mask_kwargs, ) if check_stats: @@ -244,6 +246,7 @@ def _run_dsl_graph( kv_tile: int | None = None, pack_gqa: bool | None = None, return_stats: bool = False, + stats_layout: str = "contiguous", ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: """Build, select, and execute the SM120 FROST graph engine. @@ -311,7 +314,7 @@ def _run_dsl_graph( stats_gpu = None if return_stats: assert stats is not None - stats_gpu = torch.empty(batch, heads, sequence, 1, dtype=torch.float32, device="cuda") + stats_gpu = make_dense_stats(batch, heads, sequence, stats_layout) stats.set_output(True).set_dim(stats_gpu.shape).set_stride(stats_gpu.stride()) stats.set_data_type(cudnn.data_type.FLOAT) variant_pack[stats] = stats_gpu @@ -692,6 +695,25 @@ def test_dsl_sm120_stats(mask_kwargs): _run_case(batch=2, h_q=4, h_kv=2, s_q=256, s_kv=256, head_dim=128, check_stats=True, **mask_kwargs) +@pytest.mark.L0 +@torch_fork_set_rng(seed=59) +def test_dsl_sm120_strided_stats(): + """Dense LSE is written directly through a permuted, gapped layout.""" + + _require_dsl() + batch, h_q, h_kv, sequence, head_dim = 2, 4, 2, 128, 128 + scale = 1.0 / math.sqrt(head_dim) + q = _bhsd(batch, h_q, sequence, head_dim, torch.float16) + k = _bhsd(batch, h_kv, sequence, head_dim, torch.float16) + v = _bhsd(batch, h_kv, sequence, head_dim, torch.float16) + _, contiguous_stats = _run_dsl_graph(q, k, v, scale=scale, is_causal=True, return_stats=True) + output, strided_stats = _run_dsl_graph(q, k, v, scale=scale, is_causal=True, return_stats=True, stats_layout="strided") + expected, expected_stats = _ref_sdpa_full(q, k, v, scale=scale, is_causal=True, return_stats=True) + torch.testing.assert_close(strided_stats, contiguous_stats, atol=0, rtol=0) + torch.testing.assert_close(strided_stats.squeeze(-1), expected_stats, atol=2e-2, rtol=2e-2) + torch.testing.assert_close(output.float(), expected, atol=0.1, rtol=5e-2) + + @pytest.mark.L0 @torch_fork_set_rng(seed=14) def test_dsl_sm120_stats_padded_trim(): diff --git a/test/python/sdpa/frost/test_sdpa_fwd_fp8_sm100.py b/test/python/sdpa/frost/test_sdpa_fwd_fp8_sm100.py index 56a14a57aa..d9b5551c5c 100644 --- a/test/python/sdpa/frost/test_sdpa_fwd_fp8_sm100.py +++ b/test/python/sdpa/frost/test_sdpa_fwd_fp8_sm100.py @@ -19,6 +19,7 @@ """ import math +from typing import NamedTuple import pytest import torch @@ -26,7 +27,7 @@ from test_utils import torch_fork_set_rng from cudnn.sdpa.fwd.engines import engine_name -from frost_test_utils import _SM, requires_blackwell, requires_dsl +from frost_test_utils import _SM, make_dense_stats, requires_blackwell, requires_dsl from frost_test_utils import select_engine as _select_engine # noqa: F401 @@ -46,6 +47,27 @@ _CUDNN_OTYPE = {torch.float16: "HALF", torch.bfloat16: "BFLOAT16", torch.float8_e4m3fn: "FP8_E4M3", torch.float8_e5m2: "FP8_E5M2"} +class _ReferenceWithStats(NamedTuple): + output: torch.Tensor + stats: torch.Tensor + + +class _RunResult(NamedTuple): + output: torch.Tensor + reference: torch.Tensor + amax: float + reference_amax: float + + +class _RunWithStatsResult(NamedTuple): + output: torch.Tensor + reference: torch.Tensor + amax: float + reference_amax: float + stats: torch.Tensor + reference_stats: torch.Tensor + + def _quant(x, in_key): fp8, fmax = _FP8[in_key], _FP8_MAX[in_key] dq = (x.abs().amax().clamp_min(1e-8) / fmax).item() @@ -79,14 +101,14 @@ def _ref(qd, kd, vd, *, scale, is_causal=False, bottom_right=False, swa_window=N probs = torch.softmax(ext, dim=-1) o = torch.matmul(probs[..., :s_kv], v_e) if return_stats: - return o, torch.logsumexp(ext, dim=-1) + return _ReferenceWithStats(o, torch.logsumexp(ext, dim=-1)) return o row_has_kv = torch.isfinite(scores).any(dim=-1, keepdim=True) probs = torch.softmax(scores, dim=-1) probs = torch.where(row_has_kv, probs, torch.zeros_like(probs)) o = torch.matmul(probs, v_e) if return_stats: - return o, torch.logsumexp(scores, dim=-1) + return _ReferenceWithStats(o, torch.logsumexp(scores, dim=-1)) return o @@ -110,6 +132,7 @@ def _run( d_qk=128, d_v=128, pack_gqa=None, + stats_layout="contiguous", return_lse=False, ): import cudnn @@ -127,7 +150,7 @@ def bshd(x8): Qb, Kb, Vb = bshd(Q8), bshd(K8), bshd(V8) Ob = torch.empty(B, S_q, H_q, d_v, device=dev, dtype=out_dt).transpose(1, 2) - lse = torch.empty(B, H_q, S_q, 1, device=dev, dtype=torch.float32) + lse = make_dense_stats(B, H_q, S_q, stats_layout) amax_o = torch.zeros(1, 1, 1, 1, device=dev, dtype=torch.float32) def sc(val): @@ -167,7 +190,7 @@ def _stns(): o, stats_t, _amx_s_unused, amx_o = g.sdpa_fp8(**kw) # Amax_S: not requested (engines decline graphs that declare it) o.set_output(True).set_dim(list(Ob.shape)).set_stride(list(Ob.stride())).set_data_type(getattr(cudnn.data_type, _CUDNN_OTYPE[out_dt])) if stats: - stats_t.set_output(True).set_dim([B, H_q, S_q, 1]).set_stride([H_q * S_q, S_q, 1, 1]).set_data_type(cudnn.data_type.FLOAT) + stats_t.set_output(True).set_dim([B, H_q, S_q, 1]).set_stride(list(lse.stride())).set_data_type(cudnn.data_type.FLOAT) amx_o.set_output(True).set_dim([1, 1, 1, 1]).set_stride([1, 1, 1, 1]).set_data_type(cudnn.data_type.FLOAT) g.validate() @@ -204,9 +227,9 @@ def _stns(): if return_lse: assert stats, "return_lse requires stats=True" o_ref, lse_ref = _ref(Q8.float() * dq, K8.float() * dk, V8.float() * dv, scale=scale, seq_lens_kv=seq_lens_kv, return_stats=True, **ref_kw) - return Ob, o_ref, amax_o.item(), o_ref.abs().max().item(), lse.view(B, H_q, S_q), lse_ref + return _RunWithStatsResult(Ob, o_ref, amax_o.item(), o_ref.abs().max().item(), lse.squeeze(-1), lse_ref) o_ref = _ref(Q8.float() * dq, K8.float() * dk, V8.float() * dv, scale=scale, seq_lens_kv=seq_lens_kv, **ref_kw) - return Ob, o_ref, amax_o.item(), o_ref.abs().max().item() + return _RunResult(Ob, o_ref, amax_o.item(), o_ref.abs().max().item()) def _ref_kwargs(sdpa_kwargs): @@ -222,8 +245,12 @@ def _ref_kwargs(sdpa_kwargs): return out +def _half_atol(in_key): + return 7e-2 if in_key == "e5m2" else 5e-2 + + def _check(out, o_ref, out_dt, in_key, amax_o, amax_o_ref): - atol = 7e-2 if in_key == "e5m2" else 5e-2 + atol = _half_atol(in_key) diff = (out.float() - o_ref).abs().max().item() if out_dt in (torch.float8_e4m3fn, torch.float8_e5m2): floor = (o_ref - o_ref.to(out_dt).float()).abs().max().item() @@ -245,6 +272,51 @@ def _check(out, o_ref, out_dt, in_key, amax_o, amax_o_ref): } +def _check_fp8_strided_stats(d_qk, d_v, in_key): + if torch.cuda.get_device_capability() == (10, 7) and d_qk != 128: + pytest.skip("SM107 per-tensor FP8 supports only d128") + kwargs = dict( + B=2, + H_q=4, + H_kv=2, + S_q=128, + S_kv=128, + in_key=in_key, + out_dt=torch.float16, + scale=1.0 / math.sqrt(d_qk), + sdpa_kwargs=dict(use_causal_mask=True), + d_qk=d_qk, + d_v=d_v, + return_lse=True, + ) + torch.manual_seed(59) + contiguous = _run(**kwargs, stats_layout="contiguous") + torch.manual_seed(59) + strided = _run(**kwargs, stats_layout="strided") + _check(strided.output, strided.reference, torch.float16, in_key, strided.amax, strided.reference_amax) + torch.testing.assert_close(strided.stats, contiguous.stats, rtol=0, atol=0) + torch.testing.assert_close(strided.stats, strided.reference_stats, rtol=3e-2, atol=_half_atol(in_key)) + + +@pytest.mark.L0 +@torch_fork_set_rng(seed=59) +def test_fp8_strided_stats(): + """The per-tensor FP8 L0 flavor preserves dense Stats strides.""" + _check_fp8_strided_stats(128, 128, "e4m3") + + +@pytest.mark.L1 +@pytest.mark.parametrize( + ("d_qk", "d_v", "in_key"), + [(128, 128, "e5m2"), (192, 128, "e4m3"), (192, 128, "e5m2")], + ids=["d128_e5m2", "d192_d128_e4m3", "d192_d128_e5m2"], +) +@torch_fork_set_rng(seed=59) +def test_fp8_strided_stats_other_flavors(d_qk, d_v, in_key): + """The remaining per-tensor FP8 flavors preserve dense Stats strides.""" + _check_fp8_strided_stats(d_qk, d_v, in_key) + + @pytest.mark.L0 @pytest.mark.parametrize("in_key", _INS) @pytest.mark.parametrize("mask", list(_MASKS)) diff --git a/test/python/sdpa/frost/test_sdpa_fwd_fp8_sm120.py b/test/python/sdpa/frost/test_sdpa_fwd_fp8_sm120.py index b674467126..a6db92a052 100644 --- a/test/python/sdpa/frost/test_sdpa_fwd_fp8_sm120.py +++ b/test/python/sdpa/frost/test_sdpa_fwd_fp8_sm120.py @@ -24,6 +24,7 @@ """ import math +from typing import NamedTuple import pytest import torch @@ -31,7 +32,7 @@ from test_utils import torch_fork_set_rng from cudnn.sdpa.fwd.engines import engine_name -from frost_test_utils import requires_blackwell_geforce, requires_dsl, select_engine as _select_engine, offers_engine +from frost_test_utils import make_dense_stats, requires_blackwell_geforce, requires_dsl, select_engine as _select_engine, offers_engine pytestmark = [requires_blackwell_geforce, requires_dsl] @@ -40,6 +41,15 @@ _FP8_MAX = {torch.float8_e4m3fn: _E4M3_MAX, torch.float8_e5m2: _E5M2_MAX} +class _RunResult(NamedTuple): + output: torch.Tensor + reference: torch.Tensor + amax: float + reference_amax: float + stats: torch.Tensor + reference_stats: torch.Tensor + + def _quant(x, dtype=torch.float8_e4m3fn): fmax = _FP8_MAX[dtype] dq = (x.abs().amax().clamp_min(1e-8) / fmax).item() @@ -106,6 +116,7 @@ def _run( so_val=1.0, layout="bshd", sinks=None, + stats_layout="contiguous", ): import cudnn @@ -140,7 +151,7 @@ def pad_s(x8, s): Ob = torch.zeros(B, H_q, S_q + 24, D_v, device=dev, dtype=o_dtype)[:, :, :S_q, :] else: raise ValueError(f"unknown layout {layout!r}") - lse = torch.empty(B, H_q, S_q, 1, device=dev, dtype=torch.float32) + lse = make_dense_stats(B, H_q, S_q, stats_layout) amax_o = torch.zeros(1, 1, 1, 1, device=dev, dtype=torch.float32) def sc(val): @@ -191,7 +202,7 @@ def _stns(): torch.float8_e5m2: cudnn.data_type.FP8_E5M2, }[o_dtype] o.set_output(True).set_dim(list(Ob.shape)).set_stride(list(Ob.stride())).set_data_type(o_cudnn) - stats.set_output(True).set_dim([B, H_q, S_q, 1]).set_stride([H_q * S_q, S_q, 1, 1]).set_data_type(cudnn.data_type.FLOAT) + stats.set_output(True).set_dim([B, H_q, S_q, 1]).set_stride(list(lse.stride())).set_data_type(cudnn.data_type.FLOAT) amx_o.set_output(True).set_dim([1, 1, 1, 1]).set_stride([1, 1, 1, 1]).set_data_type(cudnn.data_type.FLOAT) g.validate() @@ -218,7 +229,7 @@ def _stns(): o_ref, lse_ref = _ref(Q8.float() * dq, K8.float() * dk, V8.float() * dv, scale=scale, seq_lens_kv=seq_lens_kv, seq_lens_q=seq_lens_q, sinks=sinks, **ref_kw) # O carries Scale_O; Amax_O is the pre-scale amax (the kernel divides it # back out), so both compare against the unscaled reference. - return Ob.float() / so_val, o_ref, amax_o.item(), o_ref.abs().max().item(), lse.squeeze(-1), lse_ref + return _RunResult(Ob.float() / so_val, o_ref, amax_o.item(), o_ref.abs().max().item(), lse.squeeze(-1), lse_ref) def _ref_kwargs(sdpa_kwargs): @@ -403,6 +414,21 @@ def test_fp8_sm120_dense_flex_layout(layout): _check(*res) +@pytest.mark.L0 +@torch_fork_set_rng(seed=59) +def test_fp8_sm120_strided_stats(): + """Dense LSE is written directly through a permuted, gapped layout.""" + + scale = 1.0 / math.sqrt(128) + kwargs = dict(B=2, H_q=4, H_kv=2, S_q=128, S_kv=128, scale=scale, sdpa_kwargs=dict(use_causal_mask=True)) + torch.manual_seed(59) + contiguous = _run(**kwargs) + torch.manual_seed(59) + strided = _run(**kwargs, stats_layout="strided") + _check(*strided) + torch.testing.assert_close(strided.stats, contiguous.stats, atol=0, rtol=0) + + @pytest.mark.L0 @torch_fork_set_rng(seed=0) def test_fp8_sm120_dense_flex_offered(): diff --git a/test/python/sdpa/frost/test_sdpa_fwd_mxfp8_sm100.py b/test/python/sdpa/frost/test_sdpa_fwd_mxfp8_sm100.py index 57f5c3acc7..0fd47d3bfe 100644 --- a/test/python/sdpa/frost/test_sdpa_fwd_mxfp8_sm100.py +++ b/test/python/sdpa/frost/test_sdpa_fwd_mxfp8_sm100.py @@ -25,6 +25,7 @@ """ import math +from typing import NamedTuple import pytest import torch @@ -32,7 +33,7 @@ from test_utils import torch_fork_set_rng from cudnn.sdpa.fwd.engines import engine_name -from frost_test_utils import requires_pre_rubin_blackwell, requires_dsl +from frost_test_utils import make_dense_stats, requires_pre_rubin_blackwell, requires_dsl from frost_test_utils import select_engine as _select_engine # noqa: F401 @@ -47,6 +48,25 @@ _BLOCK = 32 +class _ReferenceWithStats(NamedTuple): + output: torch.Tensor + stats: torch.Tensor + + +class _RunResult(NamedTuple): + output: torch.Tensor + reference: torch.Tensor + amax: torch.Tensor + + +class _RunWithStatsResult(NamedTuple): + output: torch.Tensor + reference: torch.Tensor + amax: torch.Tensor + stats: torch.Tensor + reference_stats: torch.Tensor + + def _cdiv(a, b): return (a + b - 1) // b @@ -69,7 +89,7 @@ def _quantize(t, b, h, s, d, fp8, *, columnwise): return data_d, swz_d, dq, (s_pad, d_scale_pad) -def _ref(qd, kd, vd, *, scale, is_causal=False, bottom_right=False, swa_window=None, sinks=None, seq_lens_kv=None): +def _ref(qd, kd, vd, *, scale, is_causal=False, bottom_right=False, swa_window=None, sinks=None, seq_lens_kv=None, return_stats=False): """fp32 reference matching the kernel's mask + sink semantics (BHSD; GQA-aware).""" b, h_q, s_q, _ = qd.shape _, h_kv, s_kv, _ = vd.shape @@ -93,14 +113,41 @@ def _ref(qd, kd, vd, *, scale, is_causal=False, bottom_right=False, swa_window=N scores = scores.masked_fill(masked, float("-inf")) if sinks is not None: col = sinks.view(1, h_q, 1, 1).float().expand(b, h_q, s_q, 1).to(dev) - probs = torch.softmax(torch.cat([scores, col], dim=-1), dim=-1) - return torch.matmul(probs[..., :s_kv], v_e) - return torch.matmul(torch.softmax(scores, dim=-1), v_e) - - -def _run(B, H_q, H_kv, S, in_key, out_dt, *, scale, sdpa_kwargs, sink=None, stats=True, seq_lens_kv=None, d_qk=128, d_v=128): + ext = torch.cat([scores, col], dim=-1) + probs = torch.softmax(ext, dim=-1) + o = torch.matmul(probs[..., :s_kv], v_e) + if return_stats: + return _ReferenceWithStats(o, torch.logsumexp(ext, dim=-1)) + return o + o = torch.matmul(torch.softmax(scores, dim=-1), v_e) + if return_stats: + return _ReferenceWithStats(o, torch.logsumexp(scores, dim=-1)) + return o + + +def _run( + B, + H_q, + H_kv, + S, + in_key, + out_dt, + *, + scale, + sdpa_kwargs, + sink=None, + stats=True, + seq_lens_kv=None, + d_qk=128, + d_v=128, + stats_layout="contiguous", + return_lse=False, +): """Quantize, build the sdpa_mxfp8 graph, route to the frost engine, execute. - Returns (O_frost [B,H,S,D] on device, O_ref fp32 [B,H,S,D]).""" + + Returns ``_RunResult`` or, when ``return_lse`` is set, + ``_RunWithStatsResult``. + """ import cudnn dev = "cuda" @@ -117,7 +164,7 @@ def bshd(x8): Qb, Kb, Vb = bshd(Q8), bshd(K8), bshd(V8) Ob = torch.empty(B, S, H_q, d_v, device=dev, dtype=out_dt).transpose(1, 2) - lse = torch.empty(B, H_q, S, 1, device=dev, dtype=torch.float32) + lse = make_dense_stats(B, H_q, S, stats_layout) amax = torch.zeros(1, 1, 1, 1, device=dev, dtype=torch.float32) sfq_g = sfq.view(torch.uint8).reshape(B, H_q, sqp, dsc) sfk_g = sfk.view(torch.uint8).reshape(B, H_kv, skp, dsc) @@ -161,7 +208,7 @@ def _sf(dims): o, stats_t, amax_o = g.sdpa_mxfp8(**kw) o.set_output(True).set_dim(list(Ob.shape)).set_stride(list(Ob.stride())).set_data_type(otype) if stats: - stats_t.set_output(True).set_dim([B, H_q, S, 1]).set_stride([H_q * S, S, 1, 1]).set_data_type(cudnn.data_type.FLOAT) + stats_t.set_output(True).set_dim([B, H_q, S, 1]).set_stride(list(lse.stride())).set_data_type(cudnn.data_type.FLOAT) amax_o.set_output(True).set_dim([1, 1, 1, 1]).set_stride([1, 1, 1, 1]).set_data_type(cudnn.data_type.FLOAT) g.validate() @@ -183,8 +230,20 @@ def _sf(dims): ref_kwargs = {k2: v2 for k2, v2 in _ref_from_sdpa(sdpa_kwargs).items()} if sink is not None: ref_kwargs["sinks"] = sink.flatten() + if return_lse: + assert stats, "return_lse requires stats=True" + o_ref, lse_ref = _ref( + Q8.float() * dqq, + K8.float() * dqk, + V8.float() * dqv, + scale=scale, + seq_lens_kv=seq_lens_kv, + return_stats=True, + **ref_kwargs, + ) + return _RunWithStatsResult(Ob, o_ref, amax, lse.squeeze(-1), lse_ref) o_ref = _ref(Q8.float() * dqq, K8.float() * dqk, V8.float() * dqv, scale=scale, seq_lens_kv=seq_lens_kv, **ref_kwargs) - return Ob, o_ref, amax + return _RunResult(Ob, o_ref, amax) def _ref_from_sdpa(sdpa_kwargs): @@ -201,6 +260,10 @@ def _ref_from_sdpa(sdpa_kwargs): return out +def _half_atol(in_key, d_qk): + return 8e-2 if in_key == "e5m2" and d_qk > 128 else 7e-2 if in_key == "e5m2" else 5e-2 + + def _check(O, O_ref, out_dt, in_key=None, d_qk=128): """Compare frost output to fp32 ref; for FP8 output, gauge against the fp8-quant floor. @@ -209,7 +272,7 @@ def _check(O, O_ref, out_dt, in_key=None, d_qk=128): """ # D192 accumulates 50% more QK products than D128. Keep the existing D128 # threshold unchanged while allowing the measured E5M2 accumulation floor. - atol_half = 8e-2 if in_key == "e5m2" and d_qk > 128 else 7e-2 if in_key == "e5m2" else 5e-2 + atol_half = _half_atol(in_key, d_qk) diff = (O.float() - O_ref).abs().max().item() if out_dt in (torch.float8_e4m3fn, torch.float8_e5m2): floor = (O_ref - O_ref.to(out_dt).float()).abs().max().item() @@ -228,6 +291,50 @@ def _check(O, O_ref, out_dt, in_key=None, d_qk=128): } +def _check_mxfp8_strided_stats(d_qk, d_v, in_key): + if torch.cuda.get_device_capability() == (10, 7): + pytest.skip("SM107 serves per-tensor FP8 d128, not block-scaled MXFP8") + kwargs = dict( + B=2, + H_q=4, + H_kv=2, + S=128, + in_key=in_key, + out_dt=torch.float16, + scale=1.0 / math.sqrt(d_qk), + sdpa_kwargs=dict(use_causal_mask=True), + d_qk=d_qk, + d_v=d_v, + return_lse=True, + ) + torch.manual_seed(59) + contiguous = _run(**kwargs, stats_layout="contiguous") + torch.manual_seed(59) + strided = _run(**kwargs, stats_layout="strided") + _check(strided.output, strided.reference, torch.float16, in_key, d_qk=d_qk) + torch.testing.assert_close(strided.stats, contiguous.stats, rtol=0, atol=0) + torch.testing.assert_close(strided.stats, strided.reference_stats, rtol=3e-2, atol=_half_atol(in_key, d_qk)) + + +@pytest.mark.L0 +@torch_fork_set_rng(seed=59) +def test_mxfp8_strided_stats(): + """The block-scaled FP8 L0 flavor preserves dense Stats strides.""" + _check_mxfp8_strided_stats(128, 128, "e4m3") + + +@pytest.mark.L1 +@pytest.mark.parametrize( + ("d_qk", "d_v", "in_key"), + [(128, 128, "e5m2"), (192, 128, "e4m3"), (192, 128, "e5m2")], + ids=["d128_e5m2", "d192_d128_e4m3", "d192_d128_e5m2"], +) +@torch_fork_set_rng(seed=59) +def test_mxfp8_strided_stats_other_flavors(d_qk, d_v, in_key): + """The remaining block-scaled FP8 flavors preserve dense Stats strides.""" + _check_mxfp8_strided_stats(d_qk, d_v, in_key) + + @pytest.mark.L0 @pytest.mark.parametrize("in_key", _INS) @torch_fork_set_rng(seed=0) diff --git a/test/python/sdpa/frost/test_sdpa_fwd_split_kv_sm100.py b/test/python/sdpa/frost/test_sdpa_fwd_split_kv_sm100.py index be60a88bc9..66920c5b5e 100644 --- a/test/python/sdpa/frost/test_sdpa_fwd_split_kv_sm100.py +++ b/test/python/sdpa/frost/test_sdpa_fwd_split_kv_sm100.py @@ -14,6 +14,7 @@ import math import os +from typing import NamedTuple, Optional import pytest import torch @@ -28,6 +29,14 @@ ROWS_PER_CLUSTER = 2 * 128 * 2 +class _ApiCaseResult(NamedTuple): + split: int + output: torch.Tensor + reference: torch.Tensor + workspace_bytes: int + stats: Optional[torch.Tensor] + + def _ref_sdpa(q, k, v, scale, is_causal, kh): """fp32 reference over BSHD inputs; returns BSHD.""" if kh != q.shape[2]: @@ -211,6 +220,56 @@ def test_split_kv_requires_lse(): mod.compile(b=1, qh=4, kh=4, sq=128, skv=2048, d_qk=D, d_v=D, has_lse=False) +@pytest.mark.L0 +def test_split_combine_compile_keeps_partial_lse_compact_and_strides_final_lse(monkeypatch): + """Only the caller-visible Stats output inherits its non-compact layout.""" + from cudnn.sdpa.fwd.kernels import split_combine_sm100 as comb + + def fake_compact_tensor(_dtype, shape, **kwargs): + return {"kind": "compact", "shape": tuple(shape), **kwargs} + + def fake_tensor(_dtype, shape, stride, **kwargs): + return {"kind": "strided", "shape": tuple(shape), "stride": tuple(stride), **kwargs} + + captured = {} + compiled = object() + + def fake_compile(*args, **kwargs): + captured["args"] = args + captured["kwargs"] = kwargs + return compiled + + monkeypatch.setattr(comb.cute.runtime, "make_fake_compact_tensor", fake_compact_tensor) + monkeypatch.setattr(comb.cute.runtime, "make_fake_tensor", fake_tensor) + monkeypatch.setattr(comb.cute.runtime, "make_fake_stream", lambda **_kwargs: object()) + monkeypatch.setattr(comb.cute, "compile", fake_compile) + + comb.compile.cache_clear() + try: + result = comb.compile( + b=2, + h=4, + sq=5, + d_v=16, + splits=3, + has_lse=True, + lse_stride=(97, 19, 3), + ) + finally: + comb.compile.cache_clear() + + assert result is compiled + assert captured["args"][2]["kind"] == "compact" + assert captured["args"][2]["shape"] == (6, 4, 5) + assert captured["args"][2]["assumed_align"] == 16 + assert captured["args"][4] == { + "kind": "strided", + "shape": (2, 4, 5), + "stride": (97, 19, 3), + "assumed_align": 4, + } + + # --- cga1 (CTA_MMA=1) --------------------------------------------------- # # cga1 drops the collective 2-CTA MMA: one independent CTA per tile, covering @@ -859,7 +918,7 @@ def _expected_split(b, h_q, s_q, s_kv, *, rows_per_tile=512, ctas_per_tile=2, kv ) -def _api_case(b, h_q, h_kv, s_q, s_kv, *, with_lse=False, workspace=True): +def _api_case(b, h_q, h_kv, s_q, s_kv, *, with_lse=False, workspace=True, lse_layout="contiguous", split_kv=None): """Drive SdpaFwdDslSm100 the way the graph path does — the chooser's value arrives as the explicit ``split_kv`` constructor knob, exactly as ``lower_dsl_prefill`` forwards a plan's knobs; return (split, O, ref).""" @@ -874,9 +933,18 @@ def _api_case(b, h_q, h_kv, s_q, s_kv, *, with_lse=False, workspace=True): k = torch.randn(b, h_kv, s_kv, d, device=dev, dtype=torch.float16) v = torch.randn(b, h_kv, s_kv, d, device=dev, dtype=torch.float16) o = torch.zeros_like(q) - lse = torch.zeros(b, h_q, s_q, device=dev, dtype=torch.float32) if with_lse else None + lse_storage = None + if not with_lse: + lse = None + elif lse_layout == "contiguous": + lse = torch.zeros(b, h_q, s_q, device=dev, dtype=torch.float32) + elif lse_layout == "strided": + lse_storage = torch.full((s_q + 7, h_q + 2, b), -12345.0, device=dev, dtype=torch.float32) + lse = lse_storage.permute(2, 1, 0)[:, :h_q, :s_q] + else: + raise ValueError(f"unknown LSE layout {lse_layout!r}") - split_knob = _expected_split(b, h_q, s_q, s_kv) + split_knob = _expected_split(b, h_q, s_q, s_kv) if split_kv is None else split_kv api = SdpaFwdDslSm100(sample_q=q, sample_k=k, sample_v=v, sample_o=o, sample_lse=lse, split_kv=split_knob) assert api.check_support() split = api.split_kv @@ -891,8 +959,15 @@ def _api_case(b, h_q, h_kv, s_q, s_kv, *, with_lse=False, workspace=True): if h_q != h_kv: kb = kb.repeat_interleave(h_q // h_kv, dim=1) vb = vb.repeat_interleave(h_q // h_kv, dim=1) - p = torch.softmax(torch.matmul(qb, kb.transpose(-1, -2)) / math.sqrt(d), dim=-1) - return split, o.float(), torch.matmul(p, vb), ws_bytes + scores = torch.matmul(qb, kb.transpose(-1, -2)) / math.sqrt(d) + p = torch.softmax(scores, dim=-1) + if lse is not None: + torch.testing.assert_close(lse, torch.logsumexp(scores, dim=-1), rtol=3e-2, atol=5e-2) + if lse_storage is not None: + gaps = torch.ones_like(lse_storage, dtype=torch.bool) + gaps[:s_q, :h_q, :] = False + assert torch.all(lse_storage[gaps] == -12345.0), "the combine LSE store touched padding outside its declared view" + return _ApiCaseResult(split, o.float(), torch.matmul(p, vb), ws_bytes, None if lse is None else lse.clone()) @pytest.mark.L0 @@ -901,10 +976,10 @@ def test_api_splits_a_decode_shape_and_is_correct(): split, the knob delivers it, the adapter sizes its own workspace and still matches fp32. S_kv is the smallest that still splits -- the fp32 reference is O(h * s_q * s_kv) and this is L0.""" - split, got, ref, ws_bytes = _api_case(1, 8, 1, 512, 16384) - assert split == _expected_split(1, 8, 512, 16384) > 1 - assert ws_bytes > 0, "a split needs the split-major O/LSE partials in workspace" - assert (got - ref).abs().max().item() <= 2e-2 + result = _api_case(1, 8, 1, 512, 16384) + assert result.split == _expected_split(1, 8, 512, 16384) > 1 + assert result.workspace_bytes > 0, "a split needs the split-major O/LSE partials in workspace" + assert (result.output - result.reference).abs().max().item() <= 2e-2 @pytest.mark.L0 @@ -914,10 +989,10 @@ def test_api_does_not_split_a_full_chip(): so hard-coding split==1 would fail on a smaller SM100 part for a device reason rather than a policy one.""" want = _expected_split(1, 16, 2048, 8192) - split, got, ref, ws_bytes = _api_case(1, 16, 16, 2048, 8192) - assert split == want - assert (ws_bytes > 0) == (split > 1), "workspace is needed exactly when we split" - assert (got - ref).abs().max().item() <= 2e-2 + result = _api_case(1, 16, 16, 2048, 8192) + assert result.split == want + assert (result.workspace_bytes > 0) == (result.split > 1), "workspace is needed exactly when we split" + assert (result.output - result.reference).abs().max().item() <= 2e-2 @pytest.mark.L0 @@ -925,18 +1000,28 @@ def test_api_does_not_split_a_full_chip(): def test_api_split_with_and_without_workspace(workspace): """With a workspace the partials are carved from it; without one they are torch-allocated (standalone use). Same answer either way.""" - split, got, ref, _ = _api_case(1, 8, 1, 512, 16384, workspace=workspace) - assert split > 1 - assert (got - ref).abs().max().item() <= 2e-2 + result = _api_case(1, 8, 1, 512, 16384, workspace=workspace) + assert result.split > 1 + assert (result.output - result.reference).abs().max().item() <= 2e-2 @pytest.mark.L0 def test_api_split_writes_the_recombined_lse(): """A Stats output under a split comes from the combine, not from any one chunk: the per-chunk LSE is compiled in even when the caller wants none.""" - split, got, ref, _ = _api_case(1, 8, 1, 512, 16384, with_lse=True) - assert split > 1 - assert (got - ref).abs().max().item() <= 2e-2 + result = _api_case(1, 8, 1, 512, 16384, with_lse=True) + assert result.split > 1 + assert (result.output - result.reference).abs().max().item() <= 2e-2 + + +@pytest.mark.L0 +def test_api_split_writes_strided_recombined_lse(): + """The combine writes the caller's final LSE through its declared stride.""" + contiguous = _api_case(1, 8, 1, 128, 1024, with_lse=True, split_kv=2) + strided = _api_case(1, 8, 1, 128, 1024, with_lse=True, lse_layout="strided", split_kv=2) + assert strided.split == 2 + assert (strided.output - strided.reference).abs().max().item() <= 2e-2 + torch.testing.assert_close(strided.stats, contiguous.stats, atol=0, rtol=0) # --- the adapter splits the FP8 family too ---------------------------------- diff --git a/test/python/sdpa/frost/test_sdpa_fwd_split_kv_sm120.py b/test/python/sdpa/frost/test_sdpa_fwd_split_kv_sm120.py index 8af3b6bcfe..a0b1733ddc 100644 --- a/test/python/sdpa/frost/test_sdpa_fwd_split_kv_sm120.py +++ b/test/python/sdpa/frost/test_sdpa_fwd_split_kv_sm120.py @@ -12,6 +12,7 @@ """ import math +from typing import NamedTuple, Optional import pytest import torch @@ -21,6 +22,15 @@ pytestmark = [requires_dsl, pytest.mark.L0] +class _ApiCaseResult(NamedTuple): + split: int + output: torch.Tensor + reference: torch.Tensor + workspace_bytes: int + expected_split: int + stats: Optional[torch.Tensor] + + def _expected_split(api): """What the chooser asks for on THIS device, from the adapter's OWN tile geometry. SM120 runs one CTA per tile (no cluster), and the adapter may pick @@ -39,7 +49,7 @@ def _expected_split(api): ) -def _sm120_case(h_q, h_kv, s_q, s_kv, *, with_lse=False, workspace=True, causal=False): +def _sm120_case(h_q, h_kv, s_q, s_kv, *, with_lse=False, workspace=True, causal=False, lse_layout="contiguous", split_kv=None): from cudnn.sdpa.fwd.api_dsl import SdpaFwdDslSm120 if torch.cuda.get_device_capability()[0] != 12: @@ -50,13 +60,22 @@ def _sm120_case(h_q, h_kv, s_q, s_kv, *, with_lse=False, workspace=True, causal= k = torch.randn(b, h_kv, s_kv, d, device=dev, dtype=torch.float16) v = torch.randn(b, h_kv, s_kv, d, device=dev, dtype=torch.float16) o = torch.zeros_like(q) - lse = torch.zeros(b, h_q, s_q, device=dev, dtype=torch.float32) if with_lse else None + lse_storage = None + if not with_lse: + lse = None + elif lse_layout == "contiguous": + lse = torch.zeros(b, h_q, s_q, device=dev, dtype=torch.float32) + elif lse_layout == "strided": + lse_storage = torch.full((s_q + 7, h_q + 2, b), -12345.0, device=dev, dtype=torch.float32) + lse = lse_storage.permute(2, 1, 0)[:, :h_q, :s_q] + else: + raise ValueError(f"unknown LSE layout {lse_layout!r}") kw = dict(is_causal=True) if causal else {} # Probe pass: the chooser reads the adapter's own tile geometry. probe = SdpaFwdDslSm120(sample_q=q, sample_k=k, sample_v=v, sample_o=o, sample_lse=lse, **kw) assert probe.check_support() - expected = _expected_split(probe) + expected = _expected_split(probe) if split_kv is None else split_kv # Knob route: the chosen split arrives as an explicit constructor knob # (split sets ride SCHED_NATURAL — the config bars a split under the LPT # remaps a causal graph would otherwise derive). @@ -81,40 +100,54 @@ def _sm120_case(h_q, h_kv, s_q, s_kv, *, with_lse=False, workspace=True, causal= j = torch.arange(s_kv, device=scores.device).view(1, s_kv) scores = scores.masked_fill(j > i, float("-inf")) p = torch.softmax(scores, dim=-1) - return split, o.float(), torch.matmul(p, vb), ws_bytes, expected + if lse is not None: + torch.testing.assert_close(lse, torch.logsumexp(scores, dim=-1), rtol=3e-2, atol=5e-2) + if lse_storage is not None: + gaps = torch.ones_like(lse_storage, dtype=torch.bool) + gaps[:s_q, :h_q, :] = False + assert torch.all(lse_storage[gaps] == -12345.0), "the combine LSE store touched padding outside its declared view" + return _ApiCaseResult(split, o.float(), torch.matmul(p, vb), ws_bytes, expected, None if lse is None else lse.clone()) def test_sm120_splits_a_decode_shape(): """A decode shape the chooser wants split -- and the adapter must honor it.""" - split, got, ref, ws_bytes, expected = _sm120_case(8, 1, 128, 32768) - if expected == 1: + result = _sm120_case(8, 1, 128, 32768) + if result.expected_split == 1: pytest.skip("this part is small enough that the shape already fills it") - assert split == expected > 1 - assert ws_bytes > 0 - assert (got - ref).abs().max().item() <= 2e-2 + assert result.split == result.expected_split > 1 + assert result.workspace_bytes > 0 + assert (result.output - result.reference).abs().max().item() <= 2e-2 def test_sm120_does_not_split_a_full_part(): """A launch that fills the part is left alone. Whether 1024x64 fills it depends on the SM count, so the expectation comes from the chooser rather than a fixed 1 that only holds on one device.""" - split, got, ref, ws_bytes, expected = _sm120_case(64, 8, 1024, 16384) - assert split == expected - assert (ws_bytes > 0) == (split > 1), "workspace is needed exactly when we split" - assert (got - ref).abs().max().item() <= 2e-2 + result = _sm120_case(64, 8, 1024, 16384) + assert result.split == result.expected_split + assert (result.workspace_bytes > 0) == (result.split > 1), "workspace is needed exactly when we split" + assert (result.output - result.reference).abs().max().item() <= 2e-2 @pytest.mark.parametrize("workspace", [True, False], ids=["carved", "standalone"]) def test_sm120_split_with_and_without_workspace(workspace): - split, got, ref, _, expected = _sm120_case(8, 1, 128, 32768, workspace=workspace) - assert split == expected - assert (got - ref).abs().max().item() <= 2e-2 + result = _sm120_case(8, 1, 128, 32768, workspace=workspace) + assert result.split == result.expected_split + assert (result.output - result.reference).abs().max().item() <= 2e-2 def test_sm120_split_writes_the_recombined_lse(): - split, got, ref, _, expected = _sm120_case(8, 1, 128, 32768, with_lse=True) - assert split == expected - assert (got - ref).abs().max().item() <= 2e-2 + result = _sm120_case(8, 1, 128, 32768, with_lse=True) + assert result.split == result.expected_split + assert (result.output - result.reference).abs().max().item() <= 2e-2 + + +def test_sm120_split_writes_strided_recombined_lse(): + contiguous = _sm120_case(8, 1, 128, 1024, with_lse=True, split_kv=2) + strided = _sm120_case(8, 1, 128, 1024, with_lse=True, lse_layout="strided", split_kv=2) + assert strided.split == strided.expected_split == 2 + assert (strided.output - strided.reference).abs().max().item() <= 2e-2 + torch.testing.assert_close(strided.stats, contiguous.stats, atol=0, rtol=0) def test_sm120_causal_split_requires_the_natural_scheduler(): @@ -139,8 +172,8 @@ def test_sm120_causal_split_requires_the_natural_scheduler(): with pytest.raises(ValueError, match="split_kv"): bad.compile() # derived LPT + split: the config backstop rejects - split, got, ref, _, expected = _sm120_case(8, 1, 512, 8192, causal=True) - if expected == 1: + result = _sm120_case(8, 1, 512, 8192, causal=True) + if result.expected_split == 1: pytest.skip("this part is small enough that the causal shape already fills it") - assert split == expected > 1, "the causal arm must actually exercise the split" - assert (got - ref).abs().max().item() <= 2e-2 + assert result.split == result.expected_split > 1, "the causal arm must actually exercise the split" + assert (result.output - result.reference).abs().max().item() <= 2e-2 diff --git a/test/python/sdpa/frost/test_sdpa_graph_analyzer.py b/test/python/sdpa/frost/test_sdpa_graph_analyzer.py index 58e0a2f5e8..0311efa8e1 100644 --- a/test/python/sdpa/frost/test_sdpa_graph_analyzer.py +++ b/test/python/sdpa/frost/test_sdpa_graph_analyzer.py @@ -664,6 +664,17 @@ def _mk_sm120_graph(d: int = 128, **sdpa_kwargs): return g +def _mk_dense_stats_graph(stats_stride, *, stats_dim=(B, H, S, 1), stats_dtype=cudnn.data_type.FLOAT): + g = _mk_graph() + q, k, v, dims, strides = _mk_qkv(g, d=128) + o, stats = g.sdpa(name="s", q=q, k=k, v=v, attn_scale=0.1, generate_stats=True) + _finish_output(o, dims, strides) + assert stats is not None + stats.set_output(True).set_dim(stats_dim).set_stride(stats_stride) + stats.set_data_type(stats_dtype) + return g + + def test_sm120_probe_accepts_causal_swa_on_both_minors(monkeypatch): for cc in ((12, 0), (12, 1)): monkeypatch.setattr(ga, "_device_cc", lambda cc=cc: cc) @@ -813,14 +824,54 @@ def test_sm120_probe_accepts_mixed_head_dims(monkeypatch): def test_sm120_probe_accepts_stats_output(monkeypatch): monkeypatch.setattr(ga, "_device_cc", lambda: (12, 0)) - g = _mk_graph() - q, k, v, dims, strides = _mk_qkv(g, d=128) - o, stats = g.sdpa(name="s", q=q, k=k, v=v, attn_scale=0.1, generate_stats=True) - _finish_output(o, dims, strides) - assert stats is not None - stats.set_output(True).set_dim((B, H, S, 1)).set_stride((H * S, S, 1, 1)) - stats.set_data_type(cudnn.data_type.FLOAT) - assert engines.engine_name(arch="sm120") in _eligible(g) + assert engines.engine_name(arch="sm120") in _eligible(_mk_dense_stats_graph((H * S, S, 1, 1))) + + +@pytest.mark.parametrize( + ("cc", "engine"), + [ + ((8, 0), engines.engine_name(arch="sm80")), + ((10, 0), engines.engine_name(arch="sm100")), + ((12, 0), engines.engine_name(arch="sm120")), + ], + ids=["sm80", "sm100", "sm120"], +) +def test_fwd_probe_accepts_strided_stats(monkeypatch, cc, engine): + monkeypatch.setattr(ga, "_device_cc", lambda: cc) + # B is innermost, then H, then S; the extra two head rows leave a gap + # between adjacent S positions. + stats_stride = (1, B, (H + 2) * B, 1) + assert engine in _eligible(_mk_dense_stats_graph(stats_stride)) + + +@pytest.mark.parametrize( + ("cc", "engine"), + [ + ((8, 0), engines.engine_name(arch="sm80")), + ((10, 0), engines.engine_name(arch="sm100")), + ((12, 0), engines.engine_name(arch="sm120")), + ], + ids=["sm80", "sm100", "sm120"], +) +@pytest.mark.parametrize("stats_stride", [(0, S, 1, 1), (1, 1, 1, 1)], ids=["broadcast", "overlapping"]) +def test_fwd_probe_rejects_aliasing_stats(monkeypatch, cc, engine, stats_stride): + monkeypatch.setattr(ga, "_device_cc", lambda: cc) + assert engine not in _eligible(_mk_dense_stats_graph(stats_stride)) + + +@pytest.mark.parametrize( + ("stats_dim", "stats_stride", "stats_dtype", "reason"), + [ + ((B, H, S, 1), (H * S, S, 1, 1), cudnn.data_type.HALF, "stats must be fp32"), + ((B, H, S), (H * S, S, 1), cudnn.data_type.FLOAT, "stats must be (B, H_q, S_q, 1)"), + ], + ids=["dtype", "shape"], +) +def test_fwd_probe_rejects_invalid_stats_metadata(monkeypatch, stats_dim, stats_stride, stats_dtype, reason): + monkeypatch.setattr(ga, "_device_cc", lambda: (10, 0)) + facts = ga.analyze(_mk_dense_stats_graph(stats_stride, stats_dim=stats_dim, stats_dtype=stats_dtype)) + capabilities = next(spec.capabilities for spec in engines.ENGINE_SPECS if spec.name == engines.engine_name(arch="sm100")) + assert reason in engines.mismatch(capabilities, facts) def test_sm120_probe_accepts_padded_stats(monkeypatch): diff --git a/test/python/sdpa/frost/test_sdpa_sm80_frontend_integration.py b/test/python/sdpa/frost/test_sdpa_sm80_frontend_integration.py index 21b79ab68e..e424e8833f 100644 --- a/test/python/sdpa/frost/test_sdpa_sm80_frontend_integration.py +++ b/test/python/sdpa/frost/test_sdpa_sm80_frontend_integration.py @@ -35,9 +35,6 @@ def _is_sm80() -> bool: _SM80 = pytest.mark.skipif(not _is_sm80(), reason="needs an SM80 (A100) GPU") -# The default pytest.ini addopts is `-m L0`; mark the whole module so it runs. -pytestmark = pytest.mark.L0 - B, H, S, D = 2, 4, 512, 128 _HALF = cudnn.data_type.HALF _SCALE = 1.0 / math.sqrt(D) @@ -47,19 +44,21 @@ def _bshd_stride(b, h, s, d): return (s * h * d, d, h * d, 1) -def _buf(): - return torch.randn(B, S, H, D, dtype=torch.float16, device="cuda").permute(0, 2, 1, 3) +def _buf(d=D): + return torch.randn(B, S, H, d, dtype=torch.float16, device="cuda").permute(0, 2, 1, 3) -def _build_fwd_graph(**sdpa_kwargs): +def _build_fwd_graph(*, d=D, stats_stride=None, **sdpa_kwargs): g = cudnn.pygraph(io_data_type=_HALF, intermediate_data_type=cudnn.data_type.FLOAT, compute_data_type=cudnn.data_type.FLOAT) - st = _bshd_stride(B, H, S, D) - q = g.tensor(name="q", dim=(B, H, S, D), stride=st, data_type=_HALF) - k = g.tensor(name="k", dim=(B, H, S, D), stride=st, data_type=_HALF) - v = g.tensor(name="v", dim=(B, H, S, D), stride=st, data_type=_HALF) - o, stats = g.sdpa(q=q, k=k, v=v, attn_scale=_SCALE, use_causal_mask=True, generate_stats=True, **sdpa_kwargs) - o.set_output(True).set_dim((B, H, S, D)).set_stride(st).set_data_type(_HALF) + st = _bshd_stride(B, H, S, d) + q = g.tensor(name="q", dim=(B, H, S, d), stride=st, data_type=_HALF) + k = g.tensor(name="k", dim=(B, H, S, d), stride=st, data_type=_HALF) + v = g.tensor(name="v", dim=(B, H, S, d), stride=st, data_type=_HALF) + o, stats = g.sdpa(q=q, k=k, v=v, attn_scale=1.0 / math.sqrt(d), use_causal_mask=True, generate_stats=True, **sdpa_kwargs) + o.set_output(True).set_dim((B, H, S, d)).set_stride(st).set_data_type(_HALF) stats.set_output(True).set_data_type(cudnn.data_type.FLOAT) + if stats_stride is not None: + stats.set_dim((B, H, S, 1)).set_stride(stats_stride) return g, q, k, v, o, stats @@ -72,6 +71,7 @@ def _native_then_pin(g, engine): g.build_plans() +@pytest.mark.L0 def test_sm80_engines_registered(): from cudnn.sdpa.bwd.engine import FrostSdpaBwdEngines from cudnn.sdpa.fwd.engine import FrostSdpaFwdEngines @@ -88,6 +88,7 @@ def test_sm80_engines_registered(): assert is_python_engine(fwd[_FWD].engine_id) and is_python_engine(bwd[_BWD].engine_id) +@pytest.mark.L0 def test_probe_rejects_unsupported_features(): """A dropout graph must leave the SM80 forward engine ineligible for a feature (not an arch) reason — normalize device_cc so this runs anywhere.""" @@ -109,6 +110,7 @@ def test_probe_rejects_unsupported_features(): assert reason is not None and "dropout" in reason +@pytest.mark.L0 def test_direction_cross_rejection(): """The fwd engine must reject sdpa_backward graphs and vice versa.""" g, *_ = _build_fwd_graph() @@ -118,6 +120,7 @@ def test_direction_cross_rejection(): @_SM80 +@pytest.mark.L0 def test_fwd_engine_end_to_end(): g, q, k, v, o, stats = _build_fwd_graph() _native_then_pin(g, _FWD) @@ -134,7 +137,51 @@ def test_fwd_engine_end_to_end(): assert torch.isfinite(stats_buf).all() +def _check_fwd_engine_strided_stats(d): + sentinel = -12345.0 + stats_storage = torch.full((S + 7, H + 2, B), sentinel, dtype=torch.float32, device="cuda") + strided_stats_buf = stats_storage.permute(2, 1, 0)[:, :H, :S].unsqueeze(-1) + compact_stats_buf = torch.empty(B, H, S, 1, dtype=torch.float32, device="cuda") + compact_graph = _build_fwd_graph(d=d, stats_stride=compact_stats_buf.stride()) + strided_graph = _build_fwd_graph(d=d, stats_stride=strided_stats_buf.stride()) + _native_then_pin(compact_graph[0], _FWD) + _native_then_pin(strided_graph[0], _FWD) + + torch.manual_seed(0) + q_buf, k_buf, v_buf = _buf(d), _buf(d), _buf(d) + for graph_tensors, stats_buf in ((compact_graph, compact_stats_buf), (strided_graph, strided_stats_buf)): + graph, q, k, v, o, stats = graph_tensors + graph.execute({q: q_buf, k: k_buf, v: v_buf, o: torch.empty_like(q_buf), stats: stats_buf}, None) + torch.cuda.synchronize() + + scale = 1.0 / math.sqrt(d) + scores = torch.matmul(q_buf.float(), k_buf.float().transpose(-1, -2)) * scale + causal_mask = torch.ones(S, S, dtype=torch.bool, device="cuda").triu(diagonal=1) + stats_ref = torch.logsumexp(scores.masked_fill(causal_mask, float("-inf")), dim=-1) + torch.testing.assert_close(strided_stats_buf, compact_stats_buf, rtol=0, atol=0) + torch.testing.assert_close(strided_stats_buf.squeeze(-1), stats_ref, rtol=3e-2, atol=5e-2) + + gaps = torch.ones_like(stats_storage, dtype=torch.bool) + gaps[:S, :H, :] = False + assert torch.all(stats_storage[gaps] == sentinel), "the LSE store touched padding outside its declared view" + + +@_SM80 +@pytest.mark.L0 +def test_fwd_engine_strided_stats(): + """The SM80 L0 half flavor writes LSE into a permuted, gapped layout.""" + _check_fwd_engine_strided_stats(128) + + +@_SM80 +@pytest.mark.L1 +def test_fwd_engine_strided_stats_d256(): + """The SM80 D256 half flavor preserves dense Stats strides.""" + _check_fwd_engine_strided_stats(256) + + @_SM80 +@pytest.mark.L0 def test_bwd_engine_end_to_end(): # Forward via the SM80 engine to produce O / stats. g, q, k, v, o, stats = _build_fwd_graph() @@ -178,6 +225,7 @@ def test_bwd_engine_end_to_end(): @_SM80 +@pytest.mark.L0 @pytest.mark.parametrize("gqa", [1, 4], ids=["mha", "gqa4x"]) def test_fwd_engine_bhsd_contiguous_layout(gqa): """dense_flex delivery: BHSD-contiguous buffers (the test_mhas_v2 norm) @@ -211,6 +259,7 @@ def test_fwd_engine_bhsd_contiguous_layout(gqa): @_SM80 +@pytest.mark.L0 def test_bwd_engine_bhsd_contiguous_layout(): """Backward counterpart of the dense_flex layout regression.""" st = (H * S * D, S * D, D, 1) # BHSD-contiguous