Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 57 additions & 16 deletions python/cudnn/sdpa/fwd/api_dsl.py
Original file line number Diff line number Diff line change
Expand Up @@ -538,9 +538,13 @@ def _dummy(self, key: str, device: torch.device, factory: Callable[[], torch.Ten
def _checked_lse_view(self, lse_tensor: torch.Tensor) -> torch.Tensor:
"""Validate a caller-provided LSE buffer and return the kernel's (B, H_q, S_q) view.

The kernel WRITES through the returned view, so this must be a true
view: a silent ``reshape`` copy of a non-contiguous buffer would
receive the output and be dropped, leaving the caller's LSE unwritten.
The logical contract is exactly ``B*H_q*S_q`` fp32 elements. Graph
Stats commonly arrive as a rank-4 ``(B, H_q, S_q, 1)`` view, which is
reinterpreted as the declared rank-3 LSE layout without copying. The
kernel writes through the returned view, so a silent ``reshape`` copy
of a non-contiguous buffer would leave the caller's Stats unwritten.
Dense adapters therefore record ``_lse_stride`` and rebuild that
declared view directly over the caller's storage.
"""
self._value_error_if(
dtype_name(lse_tensor) != "float32",
Expand All @@ -551,11 +555,22 @@ def _checked_lse_view(self, lse_tensor: torch.Tensor) -> torch.Tensor:
lse_tensor.numel() != expected,
f"lse_tensor must have B*H_q*S_q = {expected} elements; got {lse_tensor.numel()}",
)
self._value_error_if(
not lse_tensor.is_contiguous(),
"lse_tensor must be contiguous (the kernel writes through this buffer)",
)
return lse_tensor.view(self.batch_size, self.h_q, self.s_q_max)
shape = (self.batch_size, self.h_q, self.s_q_max)
stride = getattr(self, "_lse_stride", None)
if stride is None:
self._value_error_if(
not lse_tensor.is_contiguous(),
"lse_tensor must be contiguous (the kernel writes through this buffer)",
)
return lse_tensor.view(shape)
if tuple(lse_tensor.shape) == shape and tuple(lse_tensor.stride()) == stride:
return lse_tensor
try:
return lse_tensor.as_strided(shape, stride, lse_tensor.storage_offset())
except RuntimeError as exc:
raise ValueError(
f"lse_tensor backing storage is too small for declared shape {shape}, stride {stride}, and storage_offset {lse_tensor.storage_offset()}"
) from exc

def _checked_sinks_1d(self, sinks: torch.Tensor) -> torch.Tensor:
"""Validate caller-provided sink logits and return the kernel's (H_q,) fp32 view.
Expand Down Expand Up @@ -728,6 +743,7 @@ def _initialize_implementation(self) -> None:
self.flavor: Optional[tuple[int, int]] = None
self.thd_stats_head_major = False
self.thd_stats_head_stride = 0
self._lse_stride: Optional[tuple[int, int, int]] = None
self._k_mod = None

def check_support(self) -> bool:
Expand Down Expand Up @@ -853,7 +869,12 @@ def check_support(self) -> bool:
self.thd_stats_head_major = head_major
self.thd_stats_head_stride = int(stride_h) if head_major else 0
else:
self._value_error_if(not self.lse_desc.is_contiguous(), "LSE must be contiguous on SM100 DSL")
self._value_error_if(
not dense_layout_ok((*self.lse_desc.shape, 1), (*self.lse_desc.stride, 1)),
f"LSE must use a dense-compatible B/H/S permutation or padded layout "
f"with non-broadcast, non-overlapping-by-span strides; got {self.lse_desc.stride}",
)
self._lse_stride = None if self.lse_desc.is_contiguous() else tuple(int(stride) for stride in self.lse_desc.stride)

self._value_error_if(not torch.cuda.is_available(), "CUDA must be available for SM100 DSL SDPA")
device = self.q_desc.device
Expand Down Expand Up @@ -1128,6 +1149,7 @@ def compile(self) -> None:
sq=self.s_q_max,
skv=self.s_k_max,
has_lse=(self.lse_desc is not None) or self.split_kv > 1,
lse_stride=None if self.split_kv > 1 else self._lse_stride,
)
if self._pertensor:
# ENVELOPE (per-tensor only): hand the kernel the ACTUAL head
Expand All @@ -1154,6 +1176,7 @@ def compile(self) -> None:
d_qk=self.head_dim_qk,
d_v=self.head_dim_v,
has_lse=(self.lse_desc is not None) or self.split_kv > 1,
lse_stride=None if self.split_kv > 1 else self._lse_stride,
)
self._combine_kernel = None
if self.split_kv > 1:
Expand All @@ -1173,6 +1196,7 @@ def compile(self) -> None:
dtype_o=self._combine_dtype_tag(),
has_lse=self.lse_desc is not None,
has_amax=self._fp8,
lse_stride=self._lse_stride,
)
self._logger.debug("compile completed")

Expand Down Expand Up @@ -1378,7 +1402,7 @@ def execute(
import cutlass

o_arg = O_scratch if o_needs_copy_back else O_view
lse_arg = lse_tensor.reshape(self.batch_size, self.h_q, self.s_q_max) if lse_tensor is not None else None
lse_arg = lse_tensor
if self.split_kv > 1:
# Redirect the mainloop into split-major partial slabs, then reduce
# into the caller's O/LSE with the plan-time-compiled combine pass.
Expand Down Expand Up @@ -1856,7 +1880,7 @@ def _execute_mxfp8(
sf_v_v = self._reshape_sf(sf_v, h_kv, n_kv_tiles, km.SF_SMEM_SIZE_V)

# has_lse=False (no Stats output): the store is compiled out; bind None.
lse = lse_tensor.reshape(b, h_q, sq) if lse_tensor is not None else None
lse = lse_tensor
sinks_t = (
self._checked_sinks_1d(sinks) if sinks is not None else self._dummy("sinks", device, lambda: torch.zeros(h_q, dtype=torch.float32, device=device))
)
Expand Down Expand Up @@ -2023,7 +2047,7 @@ def _execute_fp8(
O = O_scratch if o_needs_copy_back else O_view

# has_lse=False (no Stats output): the store is compiled out; bind None.
lse = lse_tensor.reshape(b, h_q, sq) if lse_tensor is not None else None
lse = lse_tensor
sinks_t = (
self._checked_sinks_1d(sinks) if sinks is not None else self._dummy("sinks", device, lambda: torch.zeros(h_q, dtype=torch.float32, device=device))
)
Expand Down Expand Up @@ -2318,6 +2342,7 @@ def _initialize_implementation(self) -> None:
self.head_dim_v: Optional[int] = None
self.thd_stats_head_major = False
self.thd_stats_head_stride = 0
self._lse_stride: Optional[tuple[int, int, int]] = None
self._k_mod = None

def check_support(self) -> bool:
Expand Down Expand Up @@ -2390,7 +2415,12 @@ def check_support(self) -> bool:
self.thd_stats_head_major = head_major
self.thd_stats_head_stride = int(stride_h) if head_major else 0
else:
self._value_error_if(not self.lse_desc.is_contiguous(), "LSE must be contiguous on SM120 DSL")
self._value_error_if(
not dense_layout_ok((*self.lse_desc.shape, 1), (*self.lse_desc.stride, 1)),
f"LSE must use a dense-compatible B/H/S permutation or padded layout "
f"with non-broadcast, non-overlapping-by-span strides; got {self.lse_desc.stride}",
)
self._lse_stride = None if self.lse_desc.is_contiguous() else tuple(int(stride) for stride in self.lse_desc.stride)

for label, val in (
("B", b),
Expand Down Expand Up @@ -2621,6 +2651,7 @@ def compile(self) -> None:
# binds no LSE buffer at all (no dummy, no allocation). A split
# REQUIRES it: the per-split LSE is the combine weight.
has_lse=(self.lse_desc is not None) or self.split_kv > 1,
lse_stride=None if self.split_kv > 1 else self._lse_stride,
)
self._combine_kernel = None
if self.split_kv > 1:
Expand All @@ -2638,6 +2669,7 @@ def compile(self) -> None:
dtype_o=self._combine_dtype_tag(),
has_lse=self.lse_desc is not None,
has_amax=False,
lse_stride=self._lse_stride,
)
self._logger.debug("compile completed")

Expand Down Expand Up @@ -3529,6 +3561,7 @@ def _initialize_implementation(self) -> None:
self.right_bound_runtime: int = 0
self._k_mod = None
self._params = None
self._lse_stride: Optional[tuple[int, int, int]] = None

# ------------------------------------------------------------------
def check_support(self) -> bool:
Expand Down Expand Up @@ -3590,7 +3623,12 @@ def check_support(self) -> bool:
if self.lse_desc is not None:
self._check_dtype(self.lse_desc, torch.float32, name="LSE")
self._check_tensor_shape(self.lse_desc, (b, h_qo, s_qo), name="LSE")
self._value_error_if(not self.lse_desc.is_contiguous(), "LSE must be contiguous on SM80")
self._value_error_if(
not dense_layout_ok((*self.lse_desc.shape, 1), (*self.lse_desc.stride, 1)),
f"LSE must use a dense-compatible B/H/S permutation or padded layout "
f"with non-broadcast, non-overlapping-by-span strides; got {self.lse_desc.stride}",
)
self._lse_stride = None if self.lse_desc.is_contiguous() else tuple(int(stride) for stride in self.lse_desc.stride)

self._not_implemented_error_if(
self.thd or self.cu_seq_q_lens or self.cu_seq_kv_lens,
Expand Down Expand Up @@ -3738,6 +3776,7 @@ def compile(self) -> None:
d=self.head_dim_qk,
swa_window=int(self.swa_window_runtime),
rope_max_s=self._rope_max_s,
lse_stride=self._lse_stride,
)
self._logger.debug("compile completed")

Expand Down Expand Up @@ -3768,15 +3807,17 @@ def execute(

# Init-time flags are compile-time specializations; execute must match
# them exactly, in both directions (Hard Rule 1).
if lse_tensor is not None and lse_tensor.ndim == 4:
lse_tensor = lse_tensor.squeeze(-1)
self._value_error_if(p.has_lse and lse_tensor is None, "compiled with a Stats output but execute() got no lse_tensor")
self._value_error_if(not p.has_lse and lse_tensor is not None, "lse_tensor provided but the plan compiled the LSE store out")
self._value_error_if(p.has_bias != (bias_tensor is not None), "bias presence must match the compiled specialization")
self._value_error_if((p.has_rope) != (rope_freqs is not None), "rope_freqs presence must match the compiled specialization")
self._value_error_if(p.has_sink != (sinks is not None), "sinks presence must match the compiled specialization")
self._value_error_if(p.has_seq_kv_lens != (seq_kv_lens is not None), "seq_kv_lens presence must match the compiled specialization")
self._value_error_if(p.has_seq_q_lens != (seq_q_lens is not None), "seq_q_lens presence must match the compiled specialization")
# Graph Stats declarations arrive as (B, H, S, 1); the kernels write
# [B, H, SQ] through the exact declared strides.
if lse_tensor is not None:
lse_tensor = self._checked_lse_view(lse_tensor)

scale_val = self.scale_softmax if (scale_softmax is None or scale_softmax == 0.0) else float(scale_softmax)
device = q_tensor.device
Expand Down
14 changes: 14 additions & 0 deletions python/cudnn/sdpa/fwd/engines.py
Original file line number Diff line number Diff line change
Expand Up @@ -454,6 +454,20 @@ def mismatch(capabilities: Capabilities, facts: "ga.SdpaGraphFacts", knobs: Opti
if facts.padded and facts.wants_stats and not facts.thd and not capabilities.padded_stats:
return "padding mask with generate_stats is not supported yet (per-batch seq_len_q LSE trim not plumbed)"

if facts.stats_t is not None and not facts.thd:
if facts.stats_t.get_data_type() != cudnn.data_type.FLOAT:
return f"stats must be fp32; got {facts.stats_t.get_data_type()}"
stats_dim = tuple(facts.stats_t.get_dim())
stats_stride = tuple(facts.stats_t.get_stride())
expected_dim = (facts.b, facts.h_q, facts.s_q, 1)
if stats_dim != expected_dim:
return f"stats must be (B, H_q, S_q, 1) = {expected_dim}; got {stats_dim}"
if not ga.dense_layout_ok(stats_dim, stats_stride):
return (
"stats must use a dense-compatible B/H/S permutation or padded layout "
f"with non-broadcast, non-overlapping-by-span strides; got {stats_stride}"
)

if capabilities.single_wave_only:
# See Capabilities.single_wave_only. 512 = TILES_Q * TILE_M * CTA_MMA
# rows per cluster; resident clusters = SM count / CTA_MMA (one CTA per
Expand Down
22 changes: 14 additions & 8 deletions python/cudnn/sdpa/fwd/kernels/prefill_d128_f16_sm100.py
Original file line number Diff line number Diff line change
Expand Up @@ -1987,8 +1987,7 @@ def _correction_warp_group(
# the chunk's O. The pair (O_s, lse_s) is everything the
# combine needs. Folds to batch_idx at SPLIT_KV == 1.
lse_batch = _partial_batch(batch_idx, split_idx, n_batch)
lse_row = lse_arr[lse_batch, row_head_idx, :]
lse_row[q_row_global] = lse_val
lse_arr[lse_batch, row_head_idx, q_row_global] = lse_val

sO_sub_base = sO[qs].base

Expand Down Expand Up @@ -2233,6 +2232,7 @@ def compile( # noqa: A001
k_stride: Optional[tuple] = None,
v_stride: Optional[tuple] = None,
o_stride: Optional[tuple] = None,
lse_stride: Optional[tuple[int, int, int]] = None,
) -> Callable:
"""Compile a kernel with ALL dims concrete to pin TMA descriptor strides at compile time.

Expand Down Expand Up @@ -2270,6 +2270,8 @@ def compile( # noqa: A001
# Each split's LSE is not optional under KV split — it IS the weight the
# combine reduces with. Without it the partials cannot be recombined.
raise ValueError("d128: split_kv > 1 requires has_lse=True (the per-split LSE drives the combine)")
if lse_stride is not None and (CFG.THD_VARLEN or SPLIT_KV > 1):
raise ValueError("dense LSE strides are not valid for THD or split-KV workspaces")
_fake_batch = 1 if CFG.THD_VARLEN else b
if CFG.THD_VARLEN:
# Dynamic packed token totals: one symbol per ragged group (Q/O and
Expand Down Expand Up @@ -2337,12 +2339,16 @@ def _fake_bshd(shape, stride, dtype=STORAGE_DTYPE, bpe=CFG.BPE):
)
else:
if lse_head_major or lse_head_stride:
raise ValueError("lse_head_major / lse_head_stride are THD-only (dense LSE is compact (B, H, Sq))")
fake_lse = cute.runtime.make_fake_compact_tensor(
cutlass.Float32,
(_lse_batch, qh, sq),
stride_order=(2, 1, 0),
assumed_align=16,
raise ValueError("lse_head_major / lse_head_stride are THD-only")
fake_lse = (
cute.runtime.make_fake_tensor(cutlass.Float32, (_lse_batch, qh, sq), lse_stride, assumed_align=4)
if lse_stride is not None
else cute.runtime.make_fake_compact_tensor(
cutlass.Float32,
(_lse_batch, qh, sq),
stride_order=(2, 1, 0),
assumed_align=16,
)
)
# Sinks tensor always part of the ABI; read only when CFG.HAS_SINK == 1 (compile-time fold).
fake_sinks = cute.runtime.make_fake_compact_tensor(
Expand Down
18 changes: 11 additions & 7 deletions python/cudnn/sdpa/fwd/kernels/prefill_d128_fp8_sm100.py
Original file line number Diff line number Diff line change
Expand Up @@ -1910,8 +1910,7 @@ def _correction_warp_group(
# TMA-STG put the chunk's O. The pair (O_s, lse_s) is everything
# the combine needs. Folds to batch_idx at SPLIT_KV == 1.
lse_batch = _partial_batch(batch_idx, split_idx, n_batch)
lse_row = lse_arr[lse_batch, row_head_idx, :]
lse_row[q_row_global] = lse_val
lse_arr[lse_batch, row_head_idx, q_row_global] = lse_val

# amax_o = max over valid rows of |o_scaled| (the fp32 pre-cast output). Divided
# by scale_o in api to give the pre-quant output amax (cuDNN FP8 ref, in-kernel).
Expand Down Expand Up @@ -2232,6 +2231,7 @@ def compile( # noqa: A001
has_lse: bool = True,
lse_head_major: bool = False,
lse_head_stride: int = 0,
lse_stride: Optional[tuple[int, int, int]] = None,
d_qk: int = CFG.TILE_K,
d_v: int = CFG.TILE_O,
) -> Callable:
Expand Down Expand Up @@ -2335,11 +2335,15 @@ def compile( # noqa: A001
else:
if lse_head_major or lse_head_stride:
raise ValueError("lse_head_major / lse_head_stride are THD-only (dense LSE is compact (B, H, Sq))")
fake_lse = cute.runtime.make_fake_compact_tensor(
cutlass.Float32,
(_lse_batch, qh, sq),
stride_order=(2, 1, 0),
assumed_align=16,
fake_lse = (
cute.runtime.make_fake_tensor(cutlass.Float32, (_lse_batch, qh, sq), lse_stride, assumed_align=4)
if lse_stride is not None and SPLIT_KV == 1
else cute.runtime.make_fake_compact_tensor(
cutlass.Float32,
(_lse_batch, qh, sq),
stride_order=(2, 1, 0),
assumed_align=16,
)
)
# Always part of the ABI; unread when CFG.HAS_SINK == 0 (compile-time fold).
fake_sinks = cute.runtime.make_fake_compact_tensor(
Expand Down
18 changes: 11 additions & 7 deletions python/cudnn/sdpa/fwd/kernels/prefill_d128_fp8_sm107.py
Original file line number Diff line number Diff line change
Expand Up @@ -2009,8 +2009,7 @@ def _correction_warp_group(
if _row_valid:
if cutlass.const_expr(lse_tensor is not None):
lse_arr = cutlass.make_array_view(lse_tensor)
lse_row = lse_arr[batch_idx, row_head_idx, :]
lse_row[q_row_global] = lse_val
lse_arr[batch_idx, row_head_idx, q_row_global] = lse_val

# amax_o = max over valid rows of |o_scaled| (the fp32 pre-cast output). Divided
# by scale_o in api to give the pre-quant output amax (cuDNN FP8 ref, in-kernel).
Expand Down Expand Up @@ -2321,6 +2320,7 @@ def compile( # noqa: A001
has_lse: bool = True,
lse_head_major: bool = False,
lse_head_stride: int = 0,
lse_stride: Optional[tuple[int, int, int]] = None,
d_qk: int = CFG.TILE_K,
d_v: int = CFG.TILE_O,
) -> Callable:
Expand Down Expand Up @@ -2414,11 +2414,15 @@ def compile( # noqa: A001
else:
if lse_head_major or lse_head_stride:
raise ValueError("lse_head_major / lse_head_stride are THD-only (dense LSE is compact (B, H, Sq))")
fake_lse = cute.runtime.make_fake_compact_tensor(
cutlass.Float32,
(b, qh, sq),
stride_order=(2, 1, 0),
assumed_align=16,
fake_lse = (
cute.runtime.make_fake_tensor(cutlass.Float32, (b, qh, sq), lse_stride, assumed_align=4)
if lse_stride is not None
else cute.runtime.make_fake_compact_tensor(
cutlass.Float32,
(b, qh, sq),
stride_order=(2, 1, 0),
assumed_align=16,
)
)
# Always part of the ABI; unread when CFG.HAS_SINK == 0 (compile-time fold).
fake_sinks = cute.runtime.make_fake_compact_tensor(
Expand Down
Loading