Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions python/cudnn/sdpa/bwd/api_dsl.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ def __init__(
sample_dv: torch.Tensor | TensorDesc,
is_causal: bool = False,
causal_bottom_right: bool = False,
window_size_left: Optional[int] = None,
scale_softmax: Optional[float] = None,
tile_m: Optional[int] = None,
tile_n: Optional[int] = None,
Expand All @@ -84,6 +85,7 @@ def __init__(

self.is_causal = bool(is_causal)
self.causal_bottom_right = bool(causal_bottom_right)
self.window_size_left = None if window_size_left is None else int(window_size_left)
self.scale_softmax = scale_softmax
self.tile_m = None if tile_m is None else int(tile_m)
self.tile_n = None if tile_n is None else int(tile_n)
Expand Down Expand Up @@ -236,6 +238,10 @@ def check_support(self) -> bool:
self.causal_bottom_right and not self.is_causal,
"causal_bottom_right requires is_causal=True",
)
self._value_error_if(
self.window_size_left is not None and self.window_size_left < 0,
f"window_size_left must be non-negative, got {self.window_size_left}",
)

self._runtime_error_if(not torch.cuda.is_available(), "CUDA is not available")
self.compute_capability = torch.cuda.get_device_capability(self.q_desc.device)
Expand Down Expand Up @@ -271,6 +277,7 @@ def compile(self) -> None:
dtype_qkv=_SM120_DTYPE_QKV_CODE[self.dtype],
is_causal=self.is_causal,
causal_top_left=self.is_causal and not self.causal_bottom_right,
window_size_left=self.window_size_left,
q_tile=self.q_tile,
kv_tile=self.kv_tile,
)
Expand Down Expand Up @@ -375,6 +382,7 @@ def sdpa_bwd_wrapper_dsl_sm120(
stats_tensor: torch.Tensor,
is_causal: bool = False,
causal_bottom_right: bool = False,
window_size_left: Optional[int] = None,
scale_softmax: Optional[float] = None,
) -> TupleDict:
"""Run SM120 SDPA backward and return ``TupleDict(dq_tensor=..., dk_tensor=..., dv_tensor=...)``."""
Expand All @@ -395,6 +403,7 @@ def sdpa_bwd_wrapper_dsl_sm120(
_tensor_signature(stats_tensor),
bool(is_causal),
bool(causal_bottom_right),
window_size_left,
scale_softmax,
)
api = _wrapper_api_cache.get(cache_key)
Expand All @@ -411,6 +420,7 @@ def sdpa_bwd_wrapper_dsl_sm120(
sample_dv=dv_tensor,
is_causal=is_causal,
causal_bottom_right=causal_bottom_right,
window_size_left=window_size_left,
scale_softmax=scale_softmax,
)
api.check_support()
Expand Down
3 changes: 3 additions & 0 deletions python/cudnn/sdpa/bwd/config_sm120.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ class TemplateParams:
dtype_qkv: int = DTYPE_FP16
is_causal: bool = False
causal_top_left: bool = False
window_size_left: int | None = None
use_pdl: bool = True
q_tile: int = 0
kv_tile: int = 0
Expand All @@ -44,6 +45,8 @@ def validate_params(params: TemplateParams) -> None:
raise ValueError(f"SM120 SDPA bwd: dtype_qkv must be DTYPE_BF16 ({DTYPE_BF16}) or DTYPE_FP16 ({DTYPE_FP16}); got {params.dtype_qkv}")
if params.causal_top_left and not params.is_causal:
raise ValueError("SM120 SDPA bwd: causal_top_left requires is_causal=True")
if params.window_size_left is not None and params.window_size_left < 0:
raise ValueError(f"SM120 SDPA bwd: window_size_left must be non-negative; got {params.window_size_left}")
if params.q_tile not in (0,) + SEQ_Q_TILES:
raise ValueError(f"SM120 SDPA bwd: q_tile must be one of {(0,) + SEQ_Q_TILES} (0 = per-head-dim default); got {params.q_tile}")
if params.kv_tile not in (0,) + SEQ_KV_TILES:
Expand Down
2 changes: 2 additions & 0 deletions python/cudnn/sdpa/bwd/engines.py
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@ def _sm120_spec() -> EngineSpec:
dtypes=frozenset({torch.float16, torch.bfloat16}),
causal=True,
bottom_right=True,
swa=True,
tile_ms=frozenset(_SM120_Q_TILES),
tile_ns=frozenset(_SM120_KV_TILES),
),
Expand Down Expand Up @@ -276,6 +277,7 @@ def _desc(geom, dtype: torch.dtype, name: str) -> "Any":
sample_dv=_desc(kv_geom, facts.dtype, "dV"),
is_causal=facts.causal,
causal_bottom_right=facts.bottom_right,
window_size_left=facts.window_left,
scale_softmax=facts.scale,
tile_m=requested.tile_m if requested is not None else None,
tile_n=requested.tile_n if requested is not None else None,
Expand Down
49 changes: 38 additions & 11 deletions python/cudnn/sdpa/bwd/kernels/bprop_f16_sm120.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
Constraints:
* Supported input dtypes: Float16 and BFloat16 (output dtype matches)
* Head dimension must be one of 32, 64, or 128
* Equal Q/KV head counts (no GQA); no dropout/alibi/local/softcap
* Equal Q/KV head counts (no GQA); no dropout/alibi/softcap
* Optional causal (top-left or bottom-right) and sliding-window masks
* Q/K/V/O/dO/dQ/dK/dV use compact BSHD storage
* LSE input is the natural-log forward stats, fp32 (B, H, SQ) contiguous
* dQ accumulation uses fp32 atomics (not bitwise deterministic)
Expand Down Expand Up @@ -294,6 +295,7 @@ def __init__(
in_dtype: Type[cutlass.Numeric] = cutlass.Float16,
is_causal: bool = False,
causal_top_left: bool = False,
window_size_left: int | None = None,
head_dim: int = 128,
use_pdl: bool = True,
q_tile: int = 0,
Expand All @@ -302,6 +304,7 @@ def __init__(
self.in_dtype = in_dtype
self.is_causal = is_causal
self.causal_top_left = bool(causal_top_left)
self.window_size_left = window_size_left
self.d = head_dim
self.use_pdl = bool(use_pdl)
self.q_tile, self.kv_tile = self.DEFAULT_TILES[head_dim]
Expand Down Expand Up @@ -436,14 +439,20 @@ def kernel(
PARTIAL_KV = (SKV % N) != 0

m_block_max = (SQ + M - 1) // M
if cutlass.const_expr(self.is_causal):
if cutlass.const_expr(self.causal_top_left):
diag_off = cutlass.Int32(0)
else:
if cutlass.const_expr(self.is_causal or self.window_size_left is not None):
if cutlass.const_expr(self.is_causal and not self.causal_top_left):
diag_off = SKV - SQ
else:
diag_off = cutlass.Int32(0)
if cutlass.const_expr(self.is_causal):
m_block_min = cute.math.max(kv_base - diag_off, cutlass.Int32(0)) // M
else:
m_block_min = cutlass.Int32(0)
if cutlass.const_expr(self.window_size_left is not None):
# q <= k - diag_off + W
last_q_row = kv_base + N - 1 - diag_off + self.window_size_left
rows_hi = cute.math.max(last_q_row + 1, cutlass.Int32(0))
m_block_max = cute.math.min(m_block_max, (rows_hi + M - 1) // M)
n_iters = cute.math.max(m_block_max - m_block_min, cutlass.Int32(0))

smem = cutlass.Array(io_dtype, self.smem_elems, space=cutlass.AddressSpace.smem, alignment=128)
Expand Down Expand Up @@ -545,6 +554,9 @@ def kernel(
math_warp = warp
math_tidx = tidx
m_block = m_block_max - 1
if cutlass.const_expr(self.window_size_left is not None):
# For bottom-right SWA, a KV tile might lie completely before every query's sliding window
m_block = cute.math.max(m_block, cutlass.Int32(0))
wm_s = math_warp % WM_SDP
wn_s = math_warp // WM_SDP
lse_r = cutlass.Array(cutlass.Float32, 2 * SDP_REPS)
Expand All @@ -569,9 +581,9 @@ def kernel(
val = inf
else:
val = (lse_ptr + lse_base + r_abs).load()
if cutlass.const_expr(self.is_causal and not self.causal_top_left):
# Fully-masked row (bottom-right, S_q > S_kv) has stats = -inf;
# +inf zeroes the row's P / grads and avoids NaN propagation.
if cutlass.const_expr((self.is_causal and not self.causal_top_left) or self.window_size_left is not None):
# A fully masked row has LSE = -inf, which can produce
# -inf - (-inf) = NaN. Use +inf to reconstruct P = 0.
if val == cutlass.Float32(float("-inf")):
val = cutlass.Float32(float("inf"))
lse_r[rep * 2 + hf] = val * cutlass.Float32(_LOG2E)
Expand Down Expand Up @@ -669,6 +681,8 @@ def kernel(
# P store to smem.
if cutlass.const_expr(self.is_causal):
do_mask_causal = (m_block * M) < (kv_base + N - diag_off)
if cutlass.const_expr(self.window_size_left is not None):
do_mask_window = kv_base < (m_block * M + M - 1 + diag_off - self.window_size_left)
neg_inf = cutlass.Float32(float("-inf"))
for rep in cutlass.range_constexpr(SDP_REPS):
for nf in cutlass.range_constexpr(SDP_NF):
Expand All @@ -692,6 +706,18 @@ def kernel(
s2 = neg_inf
if kv_a1 > r8 + diag_off:
s3 = neg_inf
if cutlass.const_expr(self.window_size_left is not None):
if do_mask_window:
lo0 = r0 + diag_off - self.window_size_left
lo8 = r8 + diag_off - self.window_size_left
if kv_a0 < lo0:
s0 = neg_inf
if kv_a1 < lo0:
s1 = neg_inf
if kv_a0 < lo8:
s2 = neg_inf
if kv_a1 < lo8:
s3 = neg_inf
if cutlass.const_expr(PARTIAL_KV):
if kv_a0 >= SKV:
s0 = neg_inf
Expand Down Expand Up @@ -835,9 +861,9 @@ def kernel(
for hf in cutlass.range_constexpr(2):
r_loc = wm_s * 16 + rep * 16 * WM_SDP + g_lane + hf * 8
val = (lse_ptr + lse_base + nq0 + r_loc).load()
if cutlass.const_expr(self.is_causal and not self.causal_top_left):
# Fully-masked row (bottom-right, S_q > S_kv) has stats = -inf;
# +inf zeroes the row's P / grads and avoids NaN propagation.
if cutlass.const_expr((self.is_causal and not self.causal_top_left) or self.window_size_left is not None):
# A fully masked row has LSE = -inf, which can produce
# -inf - (-inf) = NaN. Use +inf to reconstruct P = 0.
if val == cutlass.Float32(float("-inf")):
val = cutlass.Float32(float("inf"))
lse_r[rep * 2 + hf] = val * cutlass.Float32(_LOG2E)
Expand Down Expand Up @@ -1206,6 +1232,7 @@ def compile( # noqa: A001
in_dtype=STORAGE_DTYPE,
is_causal=PARAMS.is_causal,
causal_top_left=PARAMS.causal_top_left,
window_size_left=PARAMS.window_size_left,
head_dim=d,
use_pdl=PARAMS.use_pdl,
q_tile=PARAMS.q_tile,
Expand Down
50 changes: 47 additions & 3 deletions test/python/sdpa/frost/test_sdpa_bwd_dsl_sm120.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ def _ref_bwd(
scale: float,
is_causal: bool = False,
causal_bottom_right: bool = False,
window_size_left: int | None = None,
):
"""Reference via the canonical refs (sdpa/fp16_ref.py)."""

Expand All @@ -87,8 +88,12 @@ def _ref_bwd(

diag_align = cudnn.diagonal_alignment.BOTTOM_RIGHT if causal_bottom_right else cudnn.diagonal_alignment.TOP_LEFT
right_bound = 0 if is_causal else None
o_ref, stats_ref, _, _ = compute_ref(q, k, v, attn_scale=scale, diag_align=diag_align, right_bound=right_bound, torch_type=q.dtype)
dq, dk, dv, _, _ = compute_ref_backward(q, k, v, o_ref, do, attn_scale=scale, diag_align=diag_align, right_bound=right_bound, torch_type=q.dtype)
# The refs take the cuDNN window LENGTH; window_size_left is the offset W = L - 1.
left_bound = None if window_size_left is None else window_size_left + 1
o_ref, stats_ref, _, _ = compute_ref(q, k, v, attn_scale=scale, diag_align=diag_align, right_bound=right_bound, left_bound=left_bound, torch_type=q.dtype)
dq, dk, dv, _, _ = compute_ref_backward(
q, k, v, o_ref, do, attn_scale=scale, diag_align=diag_align, right_bound=right_bound, left_bound=left_bound, torch_type=q.dtype
)
return o_ref.to(q.dtype), stats_ref.contiguous(), dq.to(q.dtype), dk.to(q.dtype), dv.to(q.dtype)


Expand All @@ -110,6 +115,7 @@ def _run_bwd_graph(
scale: float,
is_causal: bool = False,
causal_bottom_right: bool = False,
window_size_left: int | None = None,
select: bool = True,
q_tile: int | None = None,
kv_tile: int | None = None,
Expand Down Expand Up @@ -153,6 +159,8 @@ def _run_bwd_graph(
bwd_kwargs["use_causal_mask_bottom_right"] = True
elif is_causal:
bwd_kwargs["use_causal_mask"] = True
if window_size_left is not None:
bwd_kwargs["sliding_window_length"] = window_size_left + 1

dq, dk, dv = graph.sdpa_backward(**bwd_kwargs)
dq.set_output(True).set_dim(dq_gpu.shape).set_stride(dq_gpu.stride())
Expand Down Expand Up @@ -217,6 +225,7 @@ def _run_case(
dtype: torch.dtype = torch.float16,
is_causal: bool = False,
causal_bottom_right: bool = False,
window_size_left: int | None = None,
select: bool = True,
q_tile: int | None = None,
kv_tile: int | None = None,
Expand All @@ -226,7 +235,9 @@ def _run_case(
k = _bhsd(batch, heads, s_kv, head_dim, dtype)
v = _bhsd(batch, heads, s_kv, head_dim, dtype)
do = _bhsd(batch, heads, s_q, head_dim, dtype)
o, stats, dq_ref, dk_ref, dv_ref = _ref_bwd(q, k, v, do, scale=scale, is_causal=is_causal, causal_bottom_right=causal_bottom_right)
o, stats, dq_ref, dk_ref, dv_ref = _ref_bwd(
q, k, v, do, scale=scale, is_causal=is_causal, causal_bottom_right=causal_bottom_right, window_size_left=window_size_left
)
o = _bhsd(batch, heads, s_q, head_dim, dtype, empty=True).copy_(o)
dq, dk, dv, plan_name = _run_bwd_graph(
q,
Expand All @@ -238,6 +249,7 @@ def _run_case(
scale=scale,
is_causal=is_causal,
causal_bottom_right=causal_bottom_right,
window_size_left=window_size_left,
select=select,
q_tile=q_tile,
kv_tile=kv_tile,
Expand Down Expand Up @@ -338,6 +350,38 @@ def test_sdpa_bwd_dsl_sm120_sequence_tails(mask: str):
)


@pytest.mark.L0
@torch_fork_set_rng(seed=8)
def test_sdpa_bwd_dsl_sm120_sliding_window_causal():
"""Top-left causal + sliding window (the training SWA shape)."""

_run_case(s_q=1024, s_kv=1024, head_dim=64, is_causal=True, window_size_left=127)


@pytest.mark.L0
@torch_fork_set_rng(seed=9)
def test_sdpa_bwd_dsl_sm120_sliding_window_no_causal():
"""A left window without a causal bit (band open to the right)."""

_run_case(head_dim=64, window_size_left=63)


@pytest.mark.L0
@torch_fork_set_rng(seed=10)
def test_sdpa_bwd_dsl_sm120_sliding_window_causal_br():
"""Bottom-right causal + sliding window across unequal sequence lengths."""

_run_case(s_q=384, s_kv=1024, head_dim=64, is_causal=True, causal_bottom_right=True, window_size_left=127)


@pytest.mark.L0
@torch_fork_set_rng(seed=11)
def test_sdpa_bwd_dsl_sm120_sliding_window_tails():
"""Sub-tile sliding window with non-tile-multiple sequence tails."""

_run_case(s_q=193, s_kv=257, head_dim=128, is_causal=True, window_size_left=16)


@pytest.mark.L0
@torch_fork_set_rng(seed=5)
def test_sdpa_bwd_dsl_sm120_auto_routing():
Expand Down
9 changes: 5 additions & 4 deletions test/python/sdpa/frost/test_sdpa_graph_analyzer.py
Original file line number Diff line number Diff line change
Expand Up @@ -849,8 +849,9 @@ def test_bwd_probe_causal_notches(monkeypatch):
assert _BWD_ENGINE in _bwd_eligible(_mk_bwd_graph(s_q=S // 2, use_causal_mask=True))
assert _BWD_ENGINE in _bwd_eligible(_mk_bwd_graph(s_q=2 * S, use_causal_mask=True))
assert _BWD_ENGINE in _bwd_eligible(_mk_bwd_graph(s_q=2 * S, use_causal_mask_bottom_right=True))
# Sliding window is not supported.
assert not _bwd_eligible(_mk_bwd_graph(use_causal_mask=True, sliding_window_length=64))
assert _BWD_ENGINE in _bwd_eligible(_mk_bwd_graph(use_causal_mask=True, sliding_window_length=64))
assert _BWD_ENGINE in _bwd_eligible(_mk_bwd_graph(sliding_window_length=64))
assert _BWD_ENGINE in _bwd_eligible(_mk_bwd_graph(s_q=S // 2, use_causal_mask_bottom_right=True, sliding_window_length=64))


def test_bwd_probe_rejects_deterministic(monkeypatch):
Expand Down Expand Up @@ -903,8 +904,8 @@ def test_bwd_mismatch_reason_strings(monkeypatch):
assert reason is not None and "GQA" in reason
reason = bwd_engines.mismatch(caps, _facts(_mk_bwd_graph(d=96)))
assert reason is not None and "96" in reason
reason = bwd_engines.mismatch(caps, _facts(_mk_bwd_graph(use_causal_mask=True, sliding_window_length=64)))
assert reason is not None and "sliding window" in reason
reason = bwd_engines.mismatch(caps, _facts(_mk_bwd_graph(use_causal_mask=True, use_alibi_mask=True)))
assert reason is not None and "ALiBi" in reason
reason = bwd_engines.mismatch(caps, _facts(_mk_bwd_graph(use_deterministic_algorithm=True)))
assert reason is not None and "deterministic" in reason
reason = bwd_engines.mismatch(caps, _facts(_mk_bwd_graph()), engines.SdpaFwdKnobs(tile_m=64))
Expand Down