diff --git a/include/cudnn_frontend_shim.h b/include/cudnn_frontend_shim.h index 3bdbb0bc82..521b2a4e15 100644 --- a/include/cudnn_frontend_shim.h +++ b/include/cudnn_frontend_shim.h @@ -23,6 +23,7 @@ #pragma once #include +#include #include #if defined NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING @@ -555,6 +556,231 @@ get_compiled_version(void) { return CUDNN_VERSION; } +#if CUDNN_VERSION >= 92200 +inline cudnnStatus_t +causal_conv1d_nwh_forward(cudaStream_t stream, + const void *x, + const void *weight, + const void *bias, + void *y, + int batch, + int dim, + int seq_len, + int kernel_size, + cudnnDataType_t data_type, + cudnnCausalConv1dActivation_t activation) { + auto effective_cudnn_ver = std::min(detail::get_compiled_version(), detail::get_backend_version()); + if (effective_cudnn_ver < 92400) { + return CUDNN_STATUS_NOT_SUPPORTED; + } +#if CUDNN_VERSION >= 92400 + NV_FE_CALL_TO_BACKEND(causal_conv1d_nwh_forward, + cudnnCausalConv1dNwhForward, + stream, + x, + weight, + bias, + y, + batch, + dim, + seq_len, + kernel_size, + data_type, + activation); +#else + (void)stream; + (void)x; + (void)weight; + (void)bias; + (void)y; + (void)batch; + (void)dim; + (void)seq_len; + (void)kernel_size; + (void)data_type; + (void)activation; + return CUDNN_STATUS_NOT_SUPPORTED; +#endif +} + +inline cudnnStatus_t +causal_conv1d_nwh_backward(cudaStream_t stream, + const void *x, + const void *weight, + const void *bias, + const void *dy, + void *dx, + void *dweight, + void *dbias, + int batch, + int dim, + int seq_len, + int kernel_size, + cudnnDataType_t data_type, + cudnnDataType_t dw_data_type, + cudnnCausalConv1dActivation_t activation) { + auto effective_cudnn_ver = std::min(detail::get_compiled_version(), detail::get_backend_version()); + if (effective_cudnn_ver < 92400) { + return CUDNN_STATUS_NOT_SUPPORTED; + } +#if CUDNN_VERSION >= 92400 + NV_FE_CALL_TO_BACKEND(causal_conv1d_nwh_backward, + cudnnCausalConv1dNwhBackward, + stream, + x, + weight, + bias, + dy, + dx, + dweight, + dbias, + batch, + dim, + seq_len, + kernel_size, + data_type, + dw_data_type, + activation); +#else + (void)stream; + (void)x; + (void)weight; + (void)bias; + (void)dy; + (void)dx; + (void)dweight; + (void)dbias; + (void)batch; + (void)dim; + (void)seq_len; + (void)kernel_size; + (void)data_type; + (void)dw_data_type; + (void)activation; + return CUDNN_STATUS_NOT_SUPPORTED; +#endif +} + +inline cudnnStatus_t +b2b_causal_conv1d_forward(cudaStream_t stream, + const void *x, + const void *weights_proj, + const void *weights_mixer, + const void *skip_bias, + void *y, + void *y_gated, + int batch, + int dim, + int seq_len, + int kernel_size_proj, + int kernel_size_mixer, + cudnnDataType_t data_type) { + auto effective_cudnn_ver = std::min(detail::get_compiled_version(), detail::get_backend_version()); + if (effective_cudnn_ver < 92400) { + return CUDNN_STATUS_NOT_SUPPORTED; + } +#if CUDNN_VERSION >= 92400 + NV_FE_CALL_TO_BACKEND(b2b_causal_conv1d_forward, + cudnnB2BCausalConv1dForward, + stream, + x, + weights_proj, + weights_mixer, + skip_bias, + y, + y_gated, + batch, + dim, + seq_len, + kernel_size_proj, + kernel_size_mixer, + data_type); +#else + (void)stream; + (void)x; + (void)weights_proj; + (void)weights_mixer; + (void)skip_bias; + (void)y; + (void)y_gated; + (void)batch; + (void)dim; + (void)seq_len; + (void)kernel_size_proj; + (void)kernel_size_mixer; + (void)data_type; + return CUDNN_STATUS_NOT_SUPPORTED; +#endif +} + +inline cudnnStatus_t +b2b_causal_conv1d_backward(cudaStream_t stream, + const void *x, + const void *weights_proj, + const void *weights_mixer, + const void *skip_bias, + const void *y, + const void *dy, + void *dx, + void *dweights_proj, + void *dweights_mixer, + void *dskip_bias, + int batch, + int dim, + int seq_len, + int kernel_size_proj, + int kernel_size_mixer, + cudnnDataType_t data_type, + cudnnDataType_t dw_data_type) { + auto effective_cudnn_ver = std::min(detail::get_compiled_version(), detail::get_backend_version()); + if (effective_cudnn_ver < 92400) { + return CUDNN_STATUS_NOT_SUPPORTED; + } +#if CUDNN_VERSION >= 92400 + NV_FE_CALL_TO_BACKEND(b2b_causal_conv1d_backward, + cudnnB2BCausalConv1dBackward, + stream, + x, + weights_proj, + weights_mixer, + skip_bias, + y, + dy, + dx, + dweights_proj, + dweights_mixer, + dskip_bias, + batch, + dim, + seq_len, + kernel_size_proj, + kernel_size_mixer, + data_type, + dw_data_type); +#else + (void)stream; + (void)x; + (void)weights_proj; + (void)weights_mixer; + (void)skip_bias; + (void)y; + (void)dy; + (void)dx; + (void)dweights_proj; + (void)dweights_mixer; + (void)dskip_bias; + (void)batch; + (void)dim; + (void)seq_len; + (void)kernel_size_proj; + (void)kernel_size_mixer; + (void)data_type; + (void)dw_data_type; + return CUDNN_STATUS_NOT_SUPPORTED; +#endif +} +#endif + inline std::string convert_version_to_str(size_t const version) { // The multiplier for major version pre-v9 and post-v9 are different. diff --git a/python/cudnn/__init__.py b/python/cudnn/__init__.py index af48c763fb..3ac564ef2c 100644 --- a/python/cudnn/__init__.py +++ b/python/cudnn/__init__.py @@ -47,7 +47,14 @@ def is_windows(): for symbol_name in symbols_to_import: globals()[symbol_name] = getattr(_pybind_module, symbol_name) -for _optional_symbol in ["causal_conv1d_forward", "causal_conv1d_backward"]: +for _optional_symbol in [ + "causal_conv1d_forward", + "causal_conv1d_backward", + "causal_conv1d_nwh_forward", + "causal_conv1d_nwh_backward", + "b2b_causal_conv1d_forward", + "b2b_causal_conv1d_backward", +]: if hasattr(_pybind_module, _optional_symbol): globals()[_optional_symbol] = getattr(_pybind_module, _optional_symbol) diff --git a/python/cudnn/ops/__init__.py b/python/cudnn/ops/__init__.py index 54a48c07be..5518db0975 100644 --- a/python/cudnn/ops/__init__.py +++ b/python/cudnn/ops/__init__.py @@ -1 +1 @@ -from .causal_conv1d import causal_conv1d +from .causal_conv1d import causal_conv1d, causal_conv1d_nwh, b2b_causal_conv1d diff --git a/python/cudnn/ops/causal_conv1d.py b/python/cudnn/ops/causal_conv1d.py index bab761975c..74512044bf 100644 --- a/python/cudnn/ops/causal_conv1d.py +++ b/python/cudnn/ops/causal_conv1d.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import List, Optional, Tuple import torch from torch import Tensor @@ -229,3 +229,469 @@ def causal_conv1d( if bias is None: bias = torch.zeros(weight.shape[0], device=x.device, dtype=x.dtype) return torch.ops.cudnn.causal_conv1d_fwd_primitive(x, weight, bias, activation) + + +# =========================================================================== +# NWH variant — x is (batch, seq_len, dim) +# =========================================================================== + + +# --------------------------------------------------------------------------- +# NWH Forward primitive +# --------------------------------------------------------------------------- + + +@torch.library.custom_op( + "cudnn::causal_conv1d_nwh_fwd_primitive", + mutates_args=(), + device_types="cuda", +) +def _nwh_fwd_primitive(x: Tensor, weight: Tensor, bias: Tensor, activation: str) -> Tensor: + if x.dim() != 3 or weight.dim() != 2 or bias.dim() != 1: + raise ValueError(f"Expected x(3D), weight(2D), bias(1D); got {x.shape}, {weight.shape}, {bias.shape}") + + if not (x.is_cuda and weight.is_cuda and bias.is_cuda): + raise ValueError(f"All tensors must be on CUDA: x.device={x.device}, " f"weight.device={weight.device}, bias.device={bias.device}") + if not (x.device == weight.device == bias.device): + raise ValueError(f"All tensors must be on the same device: x.device={x.device}, " f"weight.device={weight.device}, bias.device={bias.device}") + + if not (x.dtype == weight.dtype == bias.dtype): + raise TypeError(f"Dtype mismatch: x.dtype={x.dtype}, weight.dtype={weight.dtype}, " f"bias.dtype={bias.dtype} (all must match)") + + x = x.contiguous() + weight = weight.contiguous() + bias = bias.contiguous() + + batch, seq_len, dim = x.shape + kernel_size = weight.shape[0] + + if weight.shape[1] != dim: + raise ValueError(f"Channel mismatch: x has dim={dim} but weight has shape {weight.shape} " f"(expected weight.shape[1]={dim})") + + if bias.shape[0] != dim: + raise ValueError(f"Bias mismatch: x has dim={dim} but bias has shape {bias.shape} " f"(expected bias.shape[0]={dim})") + + y = torch.empty_like(x) + + import cudnn + + cudnn.causal_conv1d_nwh_forward( + torch.cuda.current_stream().cuda_stream, + x.data_ptr(), + weight.data_ptr(), + bias.data_ptr(), + y.data_ptr(), + batch, + dim, + seq_len, + kernel_size, + _dtype_to_int(x.dtype), + _activation_to_int(activation), + ) + return y + + +@torch.library.register_fake("cudnn::causal_conv1d_nwh_fwd_primitive") +def _nwh_fwd_fake(x: Tensor, weight: Tensor, bias: Tensor, activation: str) -> Tensor: + return torch.empty_like(x) + + +# --------------------------------------------------------------------------- +# NWH Backward primitive +# --------------------------------------------------------------------------- + + +@torch.library.custom_op( + "cudnn::causal_conv1d_nwh_bwd_primitive", + mutates_args=(), + device_types="cuda", +) +def _nwh_bwd_primitive(grad_out: Tensor, x: Tensor, weight: Tensor, bias: Tensor, activation: str) -> List[Tensor]: + if x.dim() != 3 or weight.dim() != 2 or bias.dim() != 1: + raise ValueError(f"Expected x(3D), weight(2D), bias(1D); got {x.shape}, {weight.shape}, {bias.shape}") + if grad_out.shape != x.shape: + raise ValueError(f"Shape mismatch: dy has shape {grad_out.shape} but x has shape {x.shape} " f"(expected dy.shape == x.shape)") + if not grad_out.is_cuda: + raise ValueError(f"grad_out must be on CUDA: grad_out.device={grad_out.device}") + if grad_out.device != x.device: + raise ValueError(f"Device mismatch: grad_out.device={grad_out.device}, x.device={x.device}") + if grad_out.dtype != x.dtype: + raise ValueError(f"Dtype mismatch: grad_out.dtype={grad_out.dtype}, x.dtype={x.dtype}") + + if not (x.is_cuda and weight.is_cuda and bias.is_cuda): + raise ValueError(f"All tensors must be on CUDA: x.device={x.device}, " f"weight.device={weight.device}, bias.device={bias.device}") + if not (x.device == weight.device == bias.device): + raise ValueError(f"All tensors must be on the same device: x.device={x.device}, " f"weight.device={weight.device}, bias.device={bias.device}") + + if not (x.dtype == weight.dtype == bias.dtype): + raise TypeError(f"Dtype mismatch: x.dtype={x.dtype}, weight.dtype={weight.dtype}, " f"bias.dtype={bias.dtype} (all must match)") + + batch, seq_len, dim = x.shape + + if weight.shape[1] != dim: + raise ValueError(f"Channel mismatch: x has dim={dim} but weight has shape {weight.shape} " f"(expected weight.shape[1]={dim})") + + if bias.shape[0] != dim: + raise ValueError(f"Bias mismatch: x has dim={dim} but bias has shape {bias.shape} " f"(expected bias.shape[0]={dim})") + + x = x.contiguous() + weight = weight.contiguous() + bias = bias.contiguous() + grad_out = grad_out.contiguous() + + kernel_size = weight.shape[0] + + dx = torch.empty_like(x) + dweight = torch.zeros(weight.shape, device=x.device, dtype=torch.float32) + dbias = torch.zeros(bias.shape, device=x.device, dtype=torch.float32) + + import cudnn + + cudnn.causal_conv1d_nwh_backward( + torch.cuda.current_stream().cuda_stream, + x.data_ptr(), + weight.data_ptr(), + bias.data_ptr(), + grad_out.data_ptr(), + dx.data_ptr(), + dweight.data_ptr(), + dbias.data_ptr(), + batch, + dim, + seq_len, + kernel_size, + _dtype_to_int(x.dtype), + _dtype_to_int(torch.float32), + _activation_to_int(activation), + ) + return [dx, dweight.to(x.dtype), dbias.to(x.dtype)] + + +@torch.library.register_fake("cudnn::causal_conv1d_nwh_bwd_primitive") +def _nwh_bwd_fake(grad_out: Tensor, x: Tensor, weight: Tensor, bias: Tensor, activation: str) -> List[Tensor]: + return [torch.empty_like(x), torch.empty_like(weight), torch.empty_like(bias)] + + +# --------------------------------------------------------------------------- +# NWH Autograd glue +# --------------------------------------------------------------------------- + + +def _nwh_setup_context(ctx, inputs, output): + x, weight, bias, activation = inputs + ctx.save_for_backward(x, weight, bias) + ctx.activation = activation + + +@torch.compiler.allow_in_graph +def _nwh_autograd_bwd(ctx, grad_out): + x, weight, bias = ctx.saved_tensors + dx, dw, db = torch.ops.cudnn.causal_conv1d_nwh_bwd_primitive(grad_out, x, weight, bias, ctx.activation) + return dx, dw, db, None + + +torch.library.register_autograd( + "cudnn::causal_conv1d_nwh_fwd_primitive", + _nwh_autograd_bwd, + setup_context=_nwh_setup_context, +) + + +# --------------------------------------------------------------------------- +# NWH Public API +# --------------------------------------------------------------------------- + + +def causal_conv1d_nwh( + x: Tensor, + weight: Tensor, + bias: Optional[Tensor] = None, + activation: str = "identity", +) -> Tensor: + r"""Depthwise causal 1D convolution (NWH layout). + + Same operation as :func:`causal_conv1d` but with NWH tensor layout:: + + y = activation(conv1d_causal(x, weight) + bias) + + Args: + x (torch.Tensor): Input tensor of shape ``(batch, seq_len, dim)``. + weight (torch.Tensor): Filter tensor of shape ``(kernel_size, dim)``. + bias (torch.Tensor | None): Optional bias of shape ``(dim,)``. + activation (str): ``"identity"`` (default) or ``"silu"``. + + Returns: + torch.Tensor: Output of shape ``(batch, seq_len, dim)``. + """ + if activation not in _ACTIVATION_TO_INT: + raise ValueError(f"Unsupported activation '{activation}'. Supported: 'identity', 'silu'.") + if bias is None: + bias = torch.zeros(weight.shape[1], device=x.device, dtype=x.dtype) + return torch.ops.cudnn.causal_conv1d_nwh_fwd_primitive(x, weight, bias, activation) + + +# =========================================================================== +# Back-to-back (B2B) causal conv1d — fused projection + gating + mixer +# =========================================================================== + + +# --------------------------------------------------------------------------- +# B2B Forward primitive +# --------------------------------------------------------------------------- + + +@torch.library.custom_op( + "cudnn::b2b_causal_conv1d_fwd_primitive", + mutates_args=(), + device_types="cuda", +) +def _b2b_fwd_primitive(x: Tensor, weights_proj: Tensor, weights_mixer: Tensor, skip_bias: Tensor) -> Tuple[Tensor, Tensor]: + if x.dim() != 3: + raise ValueError(f"Expected x(3D) with shape (batch, 3*dim, seq_len); got {x.shape}") + batch = x.shape[0] + if x.shape[1] % 3 != 0: + raise ValueError(f"Expected x.shape[1] divisible by 3; got {x.shape[1]}") + dim = x.shape[1] // 3 + seq_len = x.shape[2] + + if weights_proj.dim() != 2 or weights_mixer.dim() != 2 or skip_bias.dim() != 1: + raise ValueError( + f"Expected weights_proj(2D: 3*dim,K_proj), weights_mixer(2D: dim,K_mixer), skip_bias(1D: dim); " + f"got {weights_proj.shape}, {weights_mixer.shape}, {skip_bias.shape}" + ) + + if not (x.is_cuda and weights_proj.is_cuda and weights_mixer.is_cuda and skip_bias.is_cuda): + raise ValueError("All tensors must be on CUDA") + if not (x.device == weights_proj.device == weights_mixer.device == skip_bias.device): + raise ValueError("All tensors must be on the same device") + + if not (x.dtype == weights_proj.dtype == weights_mixer.dtype == skip_bias.dtype): + raise TypeError( + f"Dtype mismatch: x.dtype={x.dtype}, weights_proj.dtype={weights_proj.dtype}, " + f"weights_mixer.dtype={weights_mixer.dtype}, skip_bias.dtype={skip_bias.dtype}" + ) + + if weights_proj.shape[0] != 3 * dim: + raise ValueError(f"Channel mismatch: x has 3*dim={3*dim} but weights_proj has shape {weights_proj.shape}") + if weights_mixer.shape[0] != dim: + raise ValueError(f"Channel mismatch: x has dim={dim} but weights_mixer has shape {weights_mixer.shape}") + if skip_bias.shape[0] != dim: + raise ValueError(f"Channel mismatch: x has dim={dim} but skip_bias has shape {skip_bias.shape}") + + x = x.contiguous() + weights_proj = weights_proj.contiguous() + weights_mixer = weights_mixer.contiguous() + skip_bias = skip_bias.contiguous() + + kernel_size_proj = weights_proj.shape[1] + kernel_size_mixer = weights_mixer.shape[1] + + y = torch.empty(batch, dim, seq_len, device=x.device, dtype=x.dtype) + y_gated = torch.empty(batch, dim, seq_len, device=x.device, dtype=x.dtype) + + import cudnn + + cudnn.b2b_causal_conv1d_forward( + torch.cuda.current_stream().cuda_stream, + x.data_ptr(), + weights_proj.data_ptr(), + weights_mixer.data_ptr(), + skip_bias.data_ptr(), + y.data_ptr(), + y_gated.data_ptr(), + batch, + dim, + seq_len, + kernel_size_proj, + kernel_size_mixer, + _dtype_to_int(x.dtype), + ) + return y, y_gated + + +@torch.library.register_fake("cudnn::b2b_causal_conv1d_fwd_primitive") +def _b2b_fwd_fake(x: Tensor, weights_proj: Tensor, weights_mixer: Tensor, skip_bias: Tensor) -> Tuple[Tensor, Tensor]: + batch = x.shape[0] + dim = x.shape[1] // 3 + seq_len = x.shape[2] + y = torch.empty(batch, dim, seq_len, device=x.device, dtype=x.dtype) + y_gated = torch.empty(batch, dim, seq_len, device=x.device, dtype=x.dtype) + return y, y_gated + + +# --------------------------------------------------------------------------- +# B2B Backward primitive +# --------------------------------------------------------------------------- + + +@torch.library.custom_op( + "cudnn::b2b_causal_conv1d_bwd_primitive", + mutates_args=(), + device_types="cuda", +) +def _b2b_bwd_primitive( + grad_y: Tensor, + x: Tensor, + weights_proj: Tensor, + weights_mixer: Tensor, + skip_bias: Tensor, + y: Tensor, +) -> List[Tensor]: + if x.dim() != 3: + raise ValueError(f"Expected x(3D) with shape (batch, 3*dim, seq_len); got {x.shape}") + if x.shape[1] % 3 != 0: + raise ValueError(f"Expected x.shape[1] divisible by 3; got {x.shape[1]}") + batch = x.shape[0] + dim = x.shape[1] // 3 + seq_len = x.shape[2] + + if weights_proj.dim() != 2 or weights_mixer.dim() != 2 or skip_bias.dim() != 1: + raise ValueError( + f"Expected weights_proj(2D: 3*dim,K_proj), weights_mixer(2D: dim,K_mixer), skip_bias(1D: dim); " + f"got {weights_proj.shape}, {weights_mixer.shape}, {skip_bias.shape}" + ) + if y.shape != (batch, dim, seq_len): + raise ValueError(f"Shape mismatch: expected y shape {(batch, dim, seq_len)}; got {y.shape}") + if grad_y.shape != y.shape: + raise ValueError(f"Shape mismatch: grad_y {grad_y.shape} vs y {y.shape}") + if not (x.is_cuda and weights_proj.is_cuda and weights_mixer.is_cuda and skip_bias.is_cuda and y.is_cuda and grad_y.is_cuda): + raise ValueError("All tensors must be on CUDA") + if not (x.device == weights_proj.device == weights_mixer.device == skip_bias.device == y.device == grad_y.device): + raise ValueError("All tensors must be on the same device") + + if not (x.dtype == weights_proj.dtype == weights_mixer.dtype == skip_bias.dtype == y.dtype == grad_y.dtype): + raise TypeError( + f"Dtype mismatch: x.dtype={x.dtype}, weights_proj.dtype={weights_proj.dtype}, " + f"weights_mixer.dtype={weights_mixer.dtype}, skip_bias.dtype={skip_bias.dtype}, " + f"y.dtype={y.dtype}, grad_y.dtype={grad_y.dtype}" + ) + + if weights_proj.shape[0] != 3 * dim: + raise ValueError(f"Channel mismatch: x has 3*dim={3*dim} but weights_proj has shape {weights_proj.shape}") + if weights_mixer.shape[0] != dim: + raise ValueError(f"Channel mismatch: x has dim={dim} but weights_mixer has shape {weights_mixer.shape}") + if skip_bias.shape[0] != dim: + raise ValueError(f"Channel mismatch: x has dim={dim} but skip_bias has shape {skip_bias.shape}") + + x = x.contiguous() + weights_proj = weights_proj.contiguous() + weights_mixer = weights_mixer.contiguous() + skip_bias = skip_bias.contiguous() + y = y.contiguous() + grad_y = grad_y.contiguous() + + kernel_size_proj = weights_proj.shape[1] + kernel_size_mixer = weights_mixer.shape[1] + + dx = torch.empty_like(x) + dweights_proj = torch.zeros(weights_proj.shape, device=x.device, dtype=torch.float32) + dweights_mixer = torch.zeros(weights_mixer.shape, device=x.device, dtype=torch.float32) + dskip_bias = torch.zeros(skip_bias.shape, device=x.device, dtype=torch.float32) + + import cudnn + + cudnn.b2b_causal_conv1d_backward( + torch.cuda.current_stream().cuda_stream, + x.data_ptr(), + weights_proj.data_ptr(), + weights_mixer.data_ptr(), + skip_bias.data_ptr(), + y.data_ptr(), + grad_y.data_ptr(), + dx.data_ptr(), + dweights_proj.data_ptr(), + dweights_mixer.data_ptr(), + dskip_bias.data_ptr(), + batch, + dim, + seq_len, + kernel_size_proj, + kernel_size_mixer, + _dtype_to_int(x.dtype), + _dtype_to_int(torch.float32), + ) + return [ + dx, + dweights_proj.to(x.dtype), + dweights_mixer.to(x.dtype), + dskip_bias.to(x.dtype), + ] + + +@torch.library.register_fake("cudnn::b2b_causal_conv1d_bwd_primitive") +def _b2b_bwd_fake( + grad_y: Tensor, + x: Tensor, + weights_proj: Tensor, + weights_mixer: Tensor, + skip_bias: Tensor, + y: Tensor, +) -> List[Tensor]: + return [ + torch.empty_like(x), + torch.empty_like(weights_proj), + torch.empty_like(weights_mixer), + torch.empty_like(skip_bias), + ] + + +# --------------------------------------------------------------------------- +# B2B Autograd glue +# --------------------------------------------------------------------------- + + +def _b2b_setup_context(ctx, inputs, output): + x, weights_proj, weights_mixer, skip_bias = inputs + y = output[0] + ctx.save_for_backward(x, weights_proj, weights_mixer, skip_bias, y) + + +@torch.compiler.allow_in_graph +def _b2b_autograd_bwd(ctx, grad_y, grad_y_gated): + # PyTorch may pass a ZeroTensor for the discarded intermediate output. + if grad_y is not None and not torch._is_zerotensor(grad_y) and torch.count_nonzero(grad_y).item() != 0: + raise RuntimeError("Gradient for the intermediate B2B output y is not supported; use cudnn.ops.b2b_causal_conv1d") + x, weights_proj, weights_mixer, skip_bias, y = ctx.saved_tensors + dx, dwp, dwm, dsb = torch.ops.cudnn.b2b_causal_conv1d_bwd_primitive(grad_y_gated, x, weights_proj, weights_mixer, skip_bias, y) + return dx, dwp, dwm, dsb + + +torch.library.register_autograd( + "cudnn::b2b_causal_conv1d_fwd_primitive", + _b2b_autograd_bwd, + setup_context=_b2b_setup_context, +) + + +# --------------------------------------------------------------------------- +# B2B Public API +# --------------------------------------------------------------------------- + + +def b2b_causal_conv1d( + x: Tensor, + weights_proj: Tensor, + weights_mixer: Tensor, + skip_bias: Tensor, +) -> Tensor: + r"""Fused back-to-back causal conv1d: projection conv, gating, mixer conv, and post-gating. + + Computes the fused Hyena-SE block:: + + proj = causal_conv1d(x, weights_proj) # (batch, 3*dim, seq_len) + gated = proj[:, 1::3, :] * proj[:, 2::3, :] # Q * K gate + y = causal_conv1d(gated, weights_mixer) + skip_bias * gated + y_gated = y * proj[:, 0::3, :] # final * V + + Args: + x (torch.Tensor): Input tensor of shape ``(batch, 3*dim, seq_len)``. + weights_proj (torch.Tensor): Projection filter ``(3*dim, kernel_size_proj)``. + weights_mixer (torch.Tensor): Mixer filter ``(dim, kernel_size_mixer)``. + skip_bias (torch.Tensor): Skip-connection bias ``(dim,)``. + + Returns: + torch.Tensor: ``y_gated`` of shape ``(batch, dim, seq_len)`` — the + post-gated final output of the fused Hyena-SE block. + """ + _, y_gated = torch.ops.cudnn.b2b_causal_conv1d_fwd_primitive(x, weights_proj, weights_mixer, skip_bias) + return y_gated diff --git a/python/pycudnn.cpp b/python/pycudnn.cpp index 9655f691ff..6b6492f61d 100644 --- a/python/pycudnn.cpp +++ b/python/pycudnn.cpp @@ -155,6 +155,140 @@ PYBIND11_MODULE(_compiled_module, m) { if (status != 0) throw std::runtime_error("cudnnCausalConv1dBackward failed with status " + std::to_string(status)); }); + + m.def("causal_conv1d_nwh_forward", + [](std::intptr_t stream, + std::intptr_t x_ptr, + std::intptr_t weight_ptr, + std::intptr_t bias_ptr, + std::intptr_t out_ptr, + int batch, + int dim, + int seq_len, + int kernel_size, + int data_type, + int activation) { + auto status = detail::causal_conv1d_nwh_forward(reinterpret_cast(stream), + reinterpret_cast(x_ptr), + reinterpret_cast(weight_ptr), + reinterpret_cast(bias_ptr), + reinterpret_cast(out_ptr), + batch, + dim, + seq_len, + kernel_size, + static_cast(data_type), + static_cast(activation)); + if (status != 0) + throw std::runtime_error("cudnnCausalConv1dNwhForward failed with status " + std::to_string(status)); + }); + + m.def("causal_conv1d_nwh_backward", + [](std::intptr_t stream, + std::intptr_t x_ptr, + std::intptr_t weight_ptr, + std::intptr_t bias_ptr, + std::intptr_t dy_ptr, + std::intptr_t dx_ptr, + std::intptr_t dweight_ptr, + std::intptr_t dbias_ptr, + int batch, + int dim, + int seq_len, + int kernel_size, + int data_type, + int dw_data_type, + int activation) { + auto status = detail::causal_conv1d_nwh_backward(reinterpret_cast(stream), + reinterpret_cast(x_ptr), + reinterpret_cast(weight_ptr), + reinterpret_cast(bias_ptr), + reinterpret_cast(dy_ptr), + reinterpret_cast(dx_ptr), + reinterpret_cast(dweight_ptr), + reinterpret_cast(dbias_ptr), + batch, + dim, + seq_len, + kernel_size, + static_cast(data_type), + static_cast(dw_data_type), + static_cast(activation)); + if (status != 0) + throw std::runtime_error("cudnnCausalConv1dNwhBackward failed with status " + std::to_string(status)); + }); + + m.def("b2b_causal_conv1d_forward", + [](std::intptr_t stream, + std::intptr_t x_ptr, + std::intptr_t weights_proj_ptr, + std::intptr_t weights_mixer_ptr, + std::intptr_t skip_bias_ptr, + std::intptr_t y_ptr, + std::intptr_t y_gated_ptr, + int batch, + int dim, + int seq_len, + int kernel_size_proj, + int kernel_size_mixer, + int data_type) { + auto status = detail::b2b_causal_conv1d_forward(reinterpret_cast(stream), + reinterpret_cast(x_ptr), + reinterpret_cast(weights_proj_ptr), + reinterpret_cast(weights_mixer_ptr), + reinterpret_cast(skip_bias_ptr), + reinterpret_cast(y_ptr), + reinterpret_cast(y_gated_ptr), + batch, + dim, + seq_len, + kernel_size_proj, + kernel_size_mixer, + static_cast(data_type)); + if (status != 0) + throw std::runtime_error("cudnnB2BCausalConv1dForward failed with status " + std::to_string(status)); + }); + + m.def("b2b_causal_conv1d_backward", + [](std::intptr_t stream, + std::intptr_t x_ptr, + std::intptr_t weights_proj_ptr, + std::intptr_t weights_mixer_ptr, + std::intptr_t skip_bias_ptr, + std::intptr_t y_ptr, + std::intptr_t dy_ptr, + std::intptr_t dx_ptr, + std::intptr_t dweights_proj_ptr, + std::intptr_t dweights_mixer_ptr, + std::intptr_t dskip_bias_ptr, + int batch, + int dim, + int seq_len, + int kernel_size_proj, + int kernel_size_mixer, + int data_type, + int dw_data_type) { + auto status = detail::b2b_causal_conv1d_backward(reinterpret_cast(stream), + reinterpret_cast(x_ptr), + reinterpret_cast(weights_proj_ptr), + reinterpret_cast(weights_mixer_ptr), + reinterpret_cast(skip_bias_ptr), + reinterpret_cast(y_ptr), + reinterpret_cast(dy_ptr), + reinterpret_cast(dx_ptr), + reinterpret_cast(dweights_proj_ptr), + reinterpret_cast(dweights_mixer_ptr), + reinterpret_cast(dskip_bias_ptr), + batch, + dim, + seq_len, + kernel_size_proj, + kernel_size_mixer, + static_cast(data_type), + static_cast(dw_data_type)); + if (status != 0) + throw std::runtime_error("cudnnB2BCausalConv1dBackward failed with status " + std::to_string(status)); + }); #endif } diff --git a/samples/cpp/CMakeLists.txt b/samples/cpp/CMakeLists.txt index 6cb727149a..20dd247524 100644 --- a/samples/cpp/CMakeLists.txt +++ b/samples/cpp/CMakeLists.txt @@ -74,6 +74,8 @@ add_executable( membound/boolean_fusion.cpp causal_conv1d/causal_conv1d.cpp + causal_conv1d/causal_conv1d_nwh.cpp + causal_conv1d/b2b_causal_conv1d.cpp ) # target flags diff --git a/samples/cpp/causal_conv1d/b2b_causal_conv1d.cpp b/samples/cpp/causal_conv1d/b2b_causal_conv1d.cpp new file mode 100644 index 0000000000..110941bc94 --- /dev/null +++ b/samples/cpp/causal_conv1d/b2b_causal_conv1d.cpp @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include "../utils/helpers.h" + +#include + +#if defined(CUDNN_SUBQUADRATIC_OPS_H_) || __has_include() +#if !defined(CUDNN_SUBQUADRATIC_OPS_H_) +#include +#endif +#define HAS_SUBQUADRATIC_OPS 1 +#else +#define HAS_SUBQUADRATIC_OPS 0 +#endif + +TEST_CASE("B2B causal conv1d forward", "[b2b_causal_conv1d][forward]") { +#if !HAS_SUBQUADRATIC_OPS + SKIP("cudnn_subquadratic_ops.h not available"); +#elif defined(_MSC_VER) + SKIP("Causal conv1d kernels are not supported on Windows (MSVC)"); +#elif CUDNN_VERSION < 92400 + SKIP("B2B causal conv1d kernels require cuDNN 9.24.0 or newer (compiled CUDNN_VERSION >= 92400)."); +#else + if (is_arch_supported_by_cudnn() == false) { + SKIP("Architecture is not supported by current cudnn version"); + } + + int batch = 2; + int dim = 64; + int seq_len = 512; + int kernel_size_proj = 4; + int kernel_size_mixer = 7; + + cudaStream_t stream = nullptr; + + Surface x_tensor(batch * dim * 3 * seq_len); + Surface w_proj(dim * 3 * kernel_size_proj); + Surface w_mixer(dim * kernel_size_mixer); + Surface skip_bias(dim); + Surface y_tensor(batch * dim * seq_len); + Surface y_gated(batch * dim * seq_len); + + CUDNN_CHECK(cudnnB2BCausalConv1dForward(stream, + x_tensor.devPtr, + w_proj.devPtr, + w_mixer.devPtr, + skip_bias.devPtr, + y_tensor.devPtr, + y_gated.devPtr, + batch, + dim, + seq_len, + kernel_size_proj, + kernel_size_mixer, + CUDNN_DATA_HALF)); + + CUDA_CHECK(cudaDeviceSynchronize()); +#endif +} + +TEST_CASE("B2B causal conv1d backward", "[b2b_causal_conv1d][backward]") { +#if !HAS_SUBQUADRATIC_OPS + SKIP("cudnn_subquadratic_ops.h not available"); +#elif defined(_MSC_VER) + SKIP("Causal conv1d kernels are not supported on Windows (MSVC)"); +#elif CUDNN_VERSION < 92400 + SKIP("B2B causal conv1d kernels require cuDNN 9.24.0 or newer (compiled CUDNN_VERSION >= 92400)."); +#else + if (is_arch_supported_by_cudnn() == false) { + SKIP("Architecture is not supported by current cudnn version"); + } + + int batch = 2; + int dim = 64; + int seq_len = 512; + int kernel_size_proj = 4; + int kernel_size_mixer = 7; + + cudaStream_t stream = nullptr; + + Surface x_tensor(batch * dim * 3 * seq_len); + Surface w_proj(dim * 3 * kernel_size_proj); + Surface w_mixer(dim * kernel_size_mixer); + Surface skip_bias(dim); + Surface y_tensor(batch * dim * seq_len); + Surface dy_tensor(batch * dim * seq_len); + Surface dx_tensor(batch * dim * 3 * seq_len); + Surface dw_proj(dim * 3 * kernel_size_proj, 0.0f); + Surface dw_mixer(dim * kernel_size_mixer, 0.0f); + Surface dskip_bias(dim, 0.0f); + + CUDNN_CHECK(cudnnB2BCausalConv1dBackward(stream, + x_tensor.devPtr, + w_proj.devPtr, + w_mixer.devPtr, + skip_bias.devPtr, + y_tensor.devPtr, + dy_tensor.devPtr, + dx_tensor.devPtr, + dw_proj.devPtr, + dw_mixer.devPtr, + dskip_bias.devPtr, + batch, + dim, + seq_len, + kernel_size_proj, + kernel_size_mixer, + CUDNN_DATA_HALF, + CUDNN_DATA_FLOAT)); + + CUDA_CHECK(cudaDeviceSynchronize()); +#endif +} diff --git a/samples/cpp/causal_conv1d/causal_conv1d_nwh.cpp b/samples/cpp/causal_conv1d/causal_conv1d_nwh.cpp new file mode 100644 index 0000000000..c0decbaa44 --- /dev/null +++ b/samples/cpp/causal_conv1d/causal_conv1d_nwh.cpp @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include "../utils/helpers.h" + +#include + +#if defined(CUDNN_SUBQUADRATIC_OPS_H_) || __has_include() +#if !defined(CUDNN_SUBQUADRATIC_OPS_H_) +#include +#endif +#define HAS_SUBQUADRATIC_OPS 1 +#else +#define HAS_SUBQUADRATIC_OPS 0 +#endif + +TEST_CASE("Causal conv1d NWH forward", "[causal_conv1d_nwh][forward]") { +#if !HAS_SUBQUADRATIC_OPS + SKIP("cudnn_subquadratic_ops.h not available"); +#elif defined(_MSC_VER) + SKIP("Causal conv1d kernels are not supported on Windows (MSVC)"); +#elif CUDNN_VERSION < 92400 + SKIP("Causal conv1d NWH kernels require cuDNN 9.24.0 or newer (compiled CUDNN_VERSION >= 92400)."); +#else + if (is_arch_supported_by_cudnn() == false) { + SKIP("Architecture is not supported by current cudnn version"); + } + + int batch = 2; + int dim = 64; + int seq_len = 512; + int kernel_size = 4; + + cudaStream_t stream = nullptr; + + Surface x_tensor(batch * seq_len * dim); + Surface w_tensor(kernel_size * dim); + Surface bias_tensor(dim); + Surface y_tensor(batch * seq_len * dim); + + CUDNN_CHECK(cudnnCausalConv1dNwhForward(stream, + x_tensor.devPtr, + w_tensor.devPtr, + bias_tensor.devPtr, + y_tensor.devPtr, + batch, + dim, + seq_len, + kernel_size, + CUDNN_DATA_HALF, + CUDNN_CAUSAL_CONV1D_ACTIVATION_SILU)); + + CUDA_CHECK(cudaDeviceSynchronize()); +#endif +} + +TEST_CASE("Causal conv1d NWH backward", "[causal_conv1d_nwh][backward]") { +#if !HAS_SUBQUADRATIC_OPS + SKIP("cudnn_subquadratic_ops.h not available"); +#elif defined(_MSC_VER) + SKIP("Causal conv1d kernels are not supported on Windows (MSVC)"); +#elif CUDNN_VERSION < 92400 + SKIP("Causal conv1d NWH kernels require cuDNN 9.24.0 or newer (compiled CUDNN_VERSION >= 92400)."); +#else + if (is_arch_supported_by_cudnn() == false) { + SKIP("Architecture is not supported by current cudnn version"); + } + + int batch = 2; + int dim = 64; + int seq_len = 512; + int kernel_size = 4; + + cudaStream_t stream = nullptr; + + Surface x_tensor(batch * seq_len * dim); + Surface w_tensor(kernel_size * dim); + Surface bias_tensor(dim); + Surface dy_tensor(batch * seq_len * dim); + Surface dx_tensor(batch * seq_len * dim); + Surface dw_tensor(kernel_size * dim, 0.0f); + Surface dbias_tensor(dim, 0.0f); + + CUDNN_CHECK(cudnnCausalConv1dNwhBackward(stream, + x_tensor.devPtr, + w_tensor.devPtr, + bias_tensor.devPtr, + dy_tensor.devPtr, + dx_tensor.devPtr, + dw_tensor.devPtr, + dbias_tensor.devPtr, + batch, + dim, + seq_len, + kernel_size, + CUDNN_DATA_HALF, + CUDNN_DATA_FLOAT, + CUDNN_CAUSAL_CONV1D_ACTIVATION_SILU)); + + CUDA_CHECK(cudaDeviceSynchronize()); +#endif +} diff --git a/samples/python/60_causal_conv1d_forward.ipynb b/samples/python/60_causal_conv1d_forward.ipynb index 75573221fa..38e286792e 100644 --- a/samples/python/60_causal_conv1d_forward.ipynb +++ b/samples/python/60_causal_conv1d_forward.ipynb @@ -25,15 +25,15 @@ "metadata": {}, "source": [ "## Prerequisites and Setup\n", - "This notebook requires an NVIDIA GPU (Hopper or later recommended).\n", + "This notebook requires an NVIDIA GPU (Hopper or later recommended) and cuDNN 9.22.0 or later.\n", "\n", - "**Environment setup** — make sure the cuDNN runtime library and the `cudnn` Python package are discoverable before launching the notebook:\n", + "**Environment setup** \u2014 make sure the cuDNN runtime library and the `cudnn` Python package are discoverable before launching the notebook:\n", "\n", - "- **Option A – pip install:**\n", + "- **Option A \u2013 pip install:**\n", " ```bash\n", " pip install nvidia-cudnn-frontend\n", " ```\n", - "- **Option B – set paths manually:**\n", + "- **Option B \u2013 set paths manually:**\n", " ```bash\n", " export LD_LIBRARY_PATH=/path/to/cudnn/lib:${LD_LIBRARY_PATH}\n", " export PYTHONPATH=/path/to/cudnn_frontend/build_python:${PYTHONPATH}\n", @@ -47,10 +47,10 @@ "execution_count": 1, "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:50:16.218874Z", - "iopub.status.busy": "2026-04-03T17:50:16.218579Z", - "iopub.status.idle": "2026-04-03T17:50:16.222536Z", - "shell.execute_reply": "2026-04-03T17:50:16.221675Z" + "iopub.execute_input": "2026-05-18T16:42:06.985867Z", + "iopub.status.busy": "2026-05-18T16:42:06.985607Z", + "iopub.status.idle": "2026-05-18T16:42:06.991250Z", + "shell.execute_reply": "2026-05-18T16:42:06.990362Z" } }, "outputs": [], @@ -63,10 +63,10 @@ "execution_count": 2, "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:50:16.225488Z", - "iopub.status.busy": "2026-04-03T17:50:16.225217Z", - "iopub.status.idle": "2026-04-03T17:50:16.228276Z", - "shell.execute_reply": "2026-04-03T17:50:16.227445Z" + "iopub.execute_input": "2026-05-18T16:42:06.994252Z", + "iopub.status.busy": "2026-05-18T16:42:06.994010Z", + "iopub.status.idle": "2026-05-18T16:42:06.997191Z", + "shell.execute_reply": "2026-05-18T16:42:06.996324Z" } }, "outputs": [], @@ -99,10 +99,10 @@ "execution_count": 3, "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:50:16.231190Z", - "iopub.status.busy": "2026-04-03T17:50:16.230925Z", - "iopub.status.idle": "2026-04-03T17:50:20.115117Z", - "shell.execute_reply": "2026-04-03T17:50:20.114082Z" + "iopub.execute_input": "2026-05-18T16:42:06.999806Z", + "iopub.status.busy": "2026-05-18T16:42:06.999568Z", + "iopub.status.idle": "2026-05-18T16:42:09.332946Z", + "shell.execute_reply": "2026-05-18T16:42:09.332099Z" } }, "outputs": [ @@ -136,10 +136,10 @@ "execution_count": 4, "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:50:20.138732Z", - "iopub.status.busy": "2026-04-03T17:50:20.138458Z", - "iopub.status.idle": "2026-04-03T17:50:20.142364Z", - "shell.execute_reply": "2026-04-03T17:50:20.141618Z" + "iopub.execute_input": "2026-05-18T16:42:09.335852Z", + "iopub.status.busy": "2026-05-18T16:42:09.335728Z", + "iopub.status.idle": "2026-05-18T16:42:09.338913Z", + "shell.execute_reply": "2026-05-18T16:42:09.338270Z" } }, "outputs": [], @@ -185,10 +185,10 @@ "execution_count": 5, "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:50:20.144248Z", - "iopub.status.busy": "2026-04-03T17:50:20.144099Z", - "iopub.status.idle": "2026-04-03T17:50:20.331438Z", - "shell.execute_reply": "2026-04-03T17:50:20.330473Z" + "iopub.execute_input": "2026-05-18T16:42:09.341191Z", + "iopub.status.busy": "2026-05-18T16:42:09.341081Z", + "iopub.status.idle": "2026-05-18T16:42:09.583721Z", + "shell.execute_reply": "2026-05-18T16:42:09.582912Z" } }, "outputs": [ @@ -236,10 +236,10 @@ "execution_count": 6, "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:50:20.333511Z", - "iopub.status.busy": "2026-04-03T17:50:20.333360Z", - "iopub.status.idle": "2026-04-03T17:50:20.363243Z", - "shell.execute_reply": "2026-04-03T17:50:20.362726Z" + "iopub.execute_input": "2026-05-18T16:42:09.586371Z", + "iopub.status.busy": "2026-05-18T16:42:09.586250Z", + "iopub.status.idle": "2026-05-18T16:42:09.599270Z", + "shell.execute_reply": "2026-05-18T16:42:09.598789Z" } }, "outputs": [ @@ -268,10 +268,10 @@ "execution_count": 7, "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:50:20.364697Z", - "iopub.status.busy": "2026-04-03T17:50:20.364555Z", - "iopub.status.idle": "2026-04-03T17:50:23.912909Z", - "shell.execute_reply": "2026-04-03T17:50:23.911867Z" + "iopub.execute_input": "2026-05-18T16:42:09.601239Z", + "iopub.status.busy": "2026-05-18T16:42:09.601119Z", + "iopub.status.idle": "2026-05-18T16:42:16.127821Z", + "shell.execute_reply": "2026-05-18T16:42:16.126982Z" } }, "outputs": [ @@ -306,10 +306,10 @@ "execution_count": 8, "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:50:23.915428Z", - "iopub.status.busy": "2026-04-03T17:50:23.915236Z", - "iopub.status.idle": "2026-04-03T17:50:23.964477Z", - "shell.execute_reply": "2026-04-03T17:50:23.963766Z" + "iopub.execute_input": "2026-05-18T16:42:16.130485Z", + "iopub.status.busy": "2026-05-18T16:42:16.130284Z", + "iopub.status.idle": "2026-05-18T16:42:16.146390Z", + "shell.execute_reply": "2026-05-18T16:42:16.145811Z" } }, "outputs": [ @@ -318,7 +318,7 @@ "output_type": "stream", "text": [ "Max absolute difference: 6.250000e-02\n", - "Max relative difference: 3.278166e-02\n", + "Max relative difference: 3.307003e-02\n", "PASSED: cuDNN causal_conv1d forward matches reference.\n" ] } @@ -355,10 +355,10 @@ "execution_count": 9, "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:50:23.966213Z", - "iopub.status.busy": "2026-04-03T17:50:23.966006Z", - "iopub.status.idle": "2026-04-03T17:50:24.090002Z", - "shell.execute_reply": "2026-04-03T17:50:24.089403Z" + "iopub.execute_input": "2026-05-18T16:42:16.148741Z", + "iopub.status.busy": "2026-05-18T16:42:16.148609Z", + "iopub.status.idle": "2026-05-18T16:42:34.578919Z", + "shell.execute_reply": "2026-05-18T16:42:34.577976Z" } }, "outputs": [ @@ -366,11 +366,35 @@ "name": "stdout", "output_type": "stream", "text": [ - "[PASS] dtype=torch.float32 K= 4 act=silu max_abs_diff=1.9073e-06\n", + "[PASS] dtype=torch.float32 K= 4 act=silu max_abs_diff=1.9073e-06\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "[PASS] dtype=torch.float16 K= 4 act=silu max_abs_diff=7.8125e-03\n", - "[PASS] dtype=torch.bfloat16 K= 4 act=silu max_abs_diff=6.2500e-02\n", - "[PASS] dtype=torch.bfloat16 K= 3 act=silu max_abs_diff=6.2500e-02\n", - "[PASS] dtype=torch.float16 K= 7 act=silu max_abs_diff=7.8125e-03\n", + "[PASS] dtype=torch.bfloat16 K= 4 act=silu max_abs_diff=6.2500e-02\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dtype=torch.bfloat16 K= 3 act=silu max_abs_diff=6.2500e-02\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dtype=torch.float16 K= 7 act=silu max_abs_diff=7.8125e-03\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "[PASS] dtype=torch.bfloat16 K= 4 act=identity max_abs_diff=0.0000e+00\n", "\n", "All tests passed!\n" @@ -434,10 +458,10 @@ "execution_count": 10, "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:50:24.092083Z", - "iopub.status.busy": "2026-04-03T17:50:24.091922Z", - "iopub.status.idle": "2026-04-03T17:50:35.078779Z", - "shell.execute_reply": "2026-04-03T17:50:35.077808Z" + "iopub.execute_input": "2026-05-18T16:42:34.581866Z", + "iopub.status.busy": "2026-05-18T16:42:34.581738Z", + "iopub.status.idle": "2026-05-18T16:42:39.443987Z", + "shell.execute_reply": "2026-05-18T16:42:39.443382Z" } }, "outputs": [ @@ -491,7 +515,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.14.0" } }, "nbformat": 4, diff --git a/samples/python/61_causal_conv1d_backward.ipynb b/samples/python/61_causal_conv1d_backward.ipynb index a663f4f248..7eb3da1e8c 100644 --- a/samples/python/61_causal_conv1d_backward.ipynb +++ b/samples/python/61_causal_conv1d_backward.ipynb @@ -21,9 +21,9 @@ "\n", "where Activation is Identity or SiLU, this notebook computes:\n", "\n", - "- $dx$ — gradient w.r.t. input $x$\n", - "- $dw$ — gradient w.r.t. weight $w$\n", - "- $db$ — gradient w.r.t. bias $b$\n", + "- $dx$ \u2014 gradient w.r.t. input $x$\n", + "- $dw$ \u2014 gradient w.r.t. weight $w$\n", + "- $db$ \u2014 gradient w.r.t. bias $b$\n", "\n", "The `cudnn.ops.causal_conv1d` API integrates with `torch.autograd`, so backward is handled automatically via `.backward()`. We compare the cuDNN autograd result against a PyTorch reference." ] @@ -34,15 +34,15 @@ "metadata": {}, "source": [ "## Prerequisites and Setup\n", - "This notebook requires an NVIDIA GPU (Hopper or later recommended).\n", + "This notebook requires an NVIDIA GPU (Hopper or later recommended) and cuDNN 9.22.0 or later.\n", "\n", - "**Environment setup** — make sure the cuDNN runtime library and the `cudnn` Python package are discoverable before launching the notebook:\n", + "**Environment setup** \u2014 make sure the cuDNN runtime library and the `cudnn` Python package are discoverable before launching the notebook:\n", "\n", - "- **Option A – pip install:**\n", + "- **Option A \u2013 pip install:**\n", " ```bash\n", " pip install nvidia-cudnn-frontend\n", " ```\n", - "- **Option B – set paths manually:**\n", + "- **Option B \u2013 set paths manually:**\n", " ```bash\n", " export LD_LIBRARY_PATH=/path/to/cudnn/lib:${LD_LIBRARY_PATH}\n", " export PYTHONPATH=/path/to/cudnn_frontend/build_python:${PYTHONPATH}\n", @@ -57,10 +57,10 @@ "id": "3da2cd41", "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:52:29.029407Z", - "iopub.status.busy": "2026-04-03T17:52:29.029100Z", - "iopub.status.idle": "2026-04-03T17:52:29.033790Z", - "shell.execute_reply": "2026-04-03T17:52:29.032514Z" + "iopub.execute_input": "2026-05-18T16:42:46.777502Z", + "iopub.status.busy": "2026-05-18T16:42:46.777193Z", + "iopub.status.idle": "2026-05-18T16:42:46.781058Z", + "shell.execute_reply": "2026-05-18T16:42:46.780270Z" } }, "outputs": [], @@ -74,10 +74,10 @@ "id": "9a439b6e", "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:52:29.036616Z", - "iopub.status.busy": "2026-04-03T17:52:29.036341Z", - "iopub.status.idle": "2026-04-03T17:52:31.788865Z", - "shell.execute_reply": "2026-04-03T17:52:31.787996Z" + "iopub.execute_input": "2026-05-18T16:42:46.783609Z", + "iopub.status.busy": "2026-05-18T16:42:46.783364Z", + "iopub.status.idle": "2026-05-18T16:42:48.459765Z", + "shell.execute_reply": "2026-05-18T16:42:48.458945Z" } }, "outputs": [ @@ -113,10 +113,10 @@ "id": "793404f1", "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:52:31.791763Z", - "iopub.status.busy": "2026-04-03T17:52:31.791566Z", - "iopub.status.idle": "2026-04-03T17:52:31.795642Z", - "shell.execute_reply": "2026-04-03T17:52:31.794935Z" + "iopub.execute_input": "2026-05-18T16:42:48.462555Z", + "iopub.status.busy": "2026-05-18T16:42:48.462431Z", + "iopub.status.idle": "2026-05-18T16:42:48.466064Z", + "shell.execute_reply": "2026-05-18T16:42:48.465389Z" } }, "outputs": [], @@ -178,10 +178,10 @@ "id": "02c03769", "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:52:31.797778Z", - "iopub.status.busy": "2026-04-03T17:52:31.797650Z", - "iopub.status.idle": "2026-04-03T17:52:32.093898Z", - "shell.execute_reply": "2026-04-03T17:52:32.092906Z" + "iopub.execute_input": "2026-05-18T16:42:48.468435Z", + "iopub.status.busy": "2026-05-18T16:42:48.468325Z", + "iopub.status.idle": "2026-05-18T16:42:48.716085Z", + "shell.execute_reply": "2026-05-18T16:42:48.715225Z" } }, "outputs": [ @@ -234,10 +234,10 @@ "id": "a7b7e1fe", "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:52:32.096767Z", - "iopub.status.busy": "2026-04-03T17:52:32.096626Z", - "iopub.status.idle": "2026-04-03T17:52:33.422754Z", - "shell.execute_reply": "2026-04-03T17:52:33.422005Z" + "iopub.execute_input": "2026-05-18T16:42:48.718773Z", + "iopub.status.busy": "2026-05-18T16:42:48.718655Z", + "iopub.status.idle": "2026-05-18T16:42:49.282101Z", + "shell.execute_reply": "2026-05-18T16:42:49.281565Z" } }, "outputs": [ @@ -272,10 +272,10 @@ "id": "7f9e824f", "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:52:33.424733Z", - "iopub.status.busy": "2026-04-03T17:52:33.424415Z", - "iopub.status.idle": "2026-04-03T17:52:34.972397Z", - "shell.execute_reply": "2026-04-03T17:52:34.971285Z" + "iopub.execute_input": "2026-05-18T16:42:49.284356Z", + "iopub.status.busy": "2026-05-18T16:42:49.284162Z", + "iopub.status.idle": "2026-05-18T16:42:58.697622Z", + "shell.execute_reply": "2026-05-18T16:42:58.696822Z" } }, "outputs": [ @@ -323,10 +323,10 @@ "id": "42562bfc", "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:52:34.975588Z", - "iopub.status.busy": "2026-04-03T17:52:34.975287Z", - "iopub.status.idle": "2026-04-03T17:52:35.012053Z", - "shell.execute_reply": "2026-04-03T17:52:35.011095Z" + "iopub.execute_input": "2026-05-18T16:42:58.699319Z", + "iopub.status.busy": "2026-05-18T16:42:58.699203Z", + "iopub.status.idle": "2026-05-18T16:42:58.710862Z", + "shell.execute_reply": "2026-05-18T16:42:58.710345Z" } }, "outputs": [ @@ -334,9 +334,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "[PASS] dx max_abs_diff=3.1241e-02\n", - "[PASS] dweight max_abs_diff=4.9881e-01\n", - "[PASS] dbias max_abs_diff=4.9422e-01\n", + "[PASS] dx max_abs_diff=4.3987e-02\n", + "[PASS] dweight max_abs_diff=4.9988e-01\n", + "[PASS] dbias max_abs_diff=5.0961e-01\n", "\n", "PASSED: cuDNN causal_conv1d autograd backward matches reference.\n" ] @@ -377,10 +377,10 @@ "id": "3588ba20", "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:52:35.014220Z", - "iopub.status.busy": "2026-04-03T17:52:35.014061Z", - "iopub.status.idle": "2026-04-03T17:52:35.188192Z", - "shell.execute_reply": "2026-04-03T17:52:35.187375Z" + "iopub.execute_input": "2026-05-18T16:42:58.712745Z", + "iopub.status.busy": "2026-05-18T16:42:58.712616Z", + "iopub.status.idle": "2026-05-18T16:43:38.408047Z", + "shell.execute_reply": "2026-05-18T16:43:38.407106Z" } }, "outputs": [ @@ -388,18 +388,36 @@ "name": "stdout", "output_type": "stream", "text": [ - "[PASS] dtype=torch.float32 K= 4 act=silu max_abs: dx=4.2915e-06 dw=3.0518e-05 db=5.7220e-05\n", - "[PASS] dtype=torch.float16 K= 4 act=silu max_abs: dx=3.8843e-03 dw=3.1090e-02 db=2.6917e-02\n", - "[PASS] dtype=torch.bfloat16 K= 4 act=silu max_abs: dx=3.0970e-02 dw=2.4898e-01 db=2.3223e-01\n", - "[PASS] dtype=torch.bfloat16 K= 3 act=silu max_abs: dx=3.1178e-02 dw=2.8888e-01 db=2.3131e-01\n", - "[PASS] dtype=torch.float16 K= 7 act=silu max_abs: dx=5.2376e-03 dw=3.4286e-02 db=2.4826e-02\n" + "[PASS] dtype=torch.float32 K= 4 act=silu max_abs: dx=4.2915e-06 dw=3.0518e-05 db=5.7220e-05\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "[PASS] dtype=torch.bfloat16 K= 4 act=identity max_abs: dx=3.1250e-02 dw=4.8404e-01 db=2.4219e-01\n", + "[PASS] dtype=torch.float16 K= 4 act=silu max_abs: dx=3.8986e-03 dw=3.0556e-02 db=2.9228e-02\n", + "[PASS] dtype=torch.bfloat16 K= 4 act=silu max_abs: dx=3.0991e-02 dw=4.4611e-01 db=1.8277e-01\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dtype=torch.bfloat16 K= 3 act=silu max_abs: dx=3.0595e-02 dw=4.2424e-01 db=2.4666e-01\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dtype=torch.float16 K= 7 act=silu max_abs: dx=7.1621e-03 dw=3.0663e-02 db=2.9175e-02\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dtype=torch.bfloat16 K= 4 act=identity max_abs: dx=3.1162e-02 dw=3.7872e-01 db=2.4889e-01\n", "\n", "All tests passed!\n" ] @@ -482,10 +500,10 @@ "id": "390c20ff", "metadata": { "execution": { - "iopub.execute_input": "2026-04-03T17:52:35.190609Z", - "iopub.status.busy": "2026-04-03T17:52:35.190460Z", - "iopub.status.idle": "2026-04-03T17:52:41.692848Z", - "shell.execute_reply": "2026-04-03T17:52:41.691995Z" + "iopub.execute_input": "2026-05-18T16:43:38.411200Z", + "iopub.status.busy": "2026-05-18T16:43:38.410921Z", + "iopub.status.idle": "2026-05-18T16:43:44.943813Z", + "shell.execute_reply": "2026-05-18T16:43:44.943246Z" } }, "outputs": [ @@ -555,7 +573,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.14.0" } }, "nbformat": 4, diff --git a/samples/python/62_causal_conv1d_nwh_forward.ipynb b/samples/python/62_causal_conv1d_nwh_forward.ipynb new file mode 100644 index 0000000000..6ca9fda5f9 --- /dev/null +++ b/samples/python/62_causal_conv1d_nwh_forward.ipynb @@ -0,0 +1,533 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Causal Conv1D NWH Forward Operation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook shows how to compute a causal depthwise 1D convolution forward operation using cuDNN with NWH (batch, seqLen, dim) layout \u2014 also known as \"NWH\".\n", + "\n", + "$$y = \\text{Activation}(\\text{CausalConv1D}(x, w) + b)$$\n", + "\n", + "where Activation is Identity or SiLU.\n", + "\n", + "The NWH layout stores the channel (dim) dimension last, which is the natural memory order for transformer-style models where the hidden dimension is the innermost axis. The convolution uses left-only (causal) padding of size `kernel_size - 1`, and each channel is convolved independently (depthwise).\n", + "\n", + "**Layout comparison:**\n", + "\n", + "| | NHW (notebook 60) | NWH (this notebook) |\n", + "|---|---|---|\n", + "| x, y | `(batch, dim, seq_len)` | `(batch, seq_len, dim)` |\n", + "| weight | `(dim, kernel_size)` | `(kernel_size, dim)` |\n", + "| bias | `(dim,)` | `(dim,)` |" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites and Setup\n", + "This notebook requires an NVIDIA GPU (Hopper or later recommended) and cuDNN 9.24.0 or later.\n", + "\n", + "**Environment setup** \u2014 make sure the cuDNN runtime library and the `cudnn` Python package are discoverable before launching the notebook:\n", + "\n", + "- **Option A \u2013 pip install:**\n", + " ```bash\n", + " pip install nvidia-cudnn-frontend\n", + " ```\n", + "- **Option B \u2013 set paths manually:**\n", + " ```bash\n", + " export LD_LIBRARY_PATH=/path/to/cudnn/lib:${LD_LIBRARY_PATH}\n", + " export PYTHONPATH=/path/to/cudnn_frontend/build_python:${PYTHONPATH}\n", + " ```\n", + "\n", + "Adjust the paths above to match your local build or installation directory." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:43:53.298978Z", + "iopub.status.busy": "2026-05-18T16:43:53.298665Z", + "iopub.status.idle": "2026-05-18T16:43:53.303027Z", + "shell.execute_reply": "2026-05-18T16:43:53.301867Z" + } + }, + "outputs": [], + "source": [ + "# !nvidia-smi" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:43:53.305655Z", + "iopub.status.busy": "2026-05-18T16:43:53.305372Z", + "iopub.status.idle": "2026-05-18T16:43:53.308943Z", + "shell.execute_reply": "2026-05-18T16:43:53.307921Z" + } + }, + "outputs": [], + "source": [ + "# get_ipython().system('pip install nvidia-cudnn-cu12')\n", + "# get_ipython().system('pip install nvidia-cudnn-frontend')\n", + "# get_ipython().system('pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Overview\n", + "\n", + "We will perform a causal depthwise conv1d forward pass with NWH layout:\n", + "\n", + "- batch size: 2\n", + "- dim (channels): 768\n", + "- sequence length: 4096\n", + "- kernel size: 4\n", + "- activation: SiLU\n", + "- data type: BFloat16\n", + "\n", + "We compare the cuDNN result against a PyTorch reference implementation." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:43:53.311388Z", + "iopub.status.busy": "2026-05-18T16:43:53.311147Z", + "iopub.status.idle": "2026-05-18T16:43:54.868804Z", + "shell.execute_reply": "2026-05-18T16:43:54.867670Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cuDNN backend version: 92300\n" + ] + } + ], + "source": [ + "import torch\n", + "import torch.nn.functional as F\n", + "import cudnn\n", + "\n", + "print(\"cuDNN backend version:\", cudnn.backend_version())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Reference Implementation\n", + "\n", + "A simple PyTorch reference for causal depthwise conv1d + SiLU with NWH layout." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:43:54.871258Z", + "iopub.status.busy": "2026-05-18T16:43:54.871132Z", + "iopub.status.idle": "2026-05-18T16:43:54.874531Z", + "shell.execute_reply": "2026-05-18T16:43:54.873832Z" + } + }, + "outputs": [], + "source": [ + "def causal_conv1d_nwh_ref(x, weight, bias, activation=\"silu\"):\n", + " \"\"\"Reference: causal depthwise conv1d for NWH layout using PyTorch ops.\n", + "\n", + " Args:\n", + " x: (batch, seq_len, dim)\n", + " weight: (kernel_size, dim)\n", + " bias: (dim,)\n", + " activation: 'silu' or 'identity'\n", + "\n", + " Returns:\n", + " y: (batch, seq_len, dim)\n", + " \"\"\"\n", + " batch, seq_len, dim = x.shape\n", + " kernel_size = weight.shape[0]\n", + "\n", + " # Transpose to NHW for PyTorch conv1d\n", + " x_nhw = x.transpose(1, 2) # (batch, dim, seq_len)\n", + "\n", + " # NWH weight (kernel_size, dim) \u2192 NHW weight (dim, 1, kernel_size) for groups=dim\n", + " w_nhw = weight.T.unsqueeze(1) # (dim, 1, kernel_size)\n", + "\n", + " # Causal padding: (kernel_size - 1) on the left, 0 on the right\n", + " x_padded = F.pad(x_nhw, (kernel_size - 1, 0))\n", + " y = F.conv1d(x_padded, w_nhw, bias=bias, groups=dim)\n", + "\n", + " if activation == \"silu\":\n", + " y = F.silu(y)\n", + "\n", + " return y.transpose(1, 2) # Back to NWH" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Tensors" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:43:54.876574Z", + "iopub.status.busy": "2026-05-18T16:43:54.876461Z", + "iopub.status.idle": "2026-05-18T16:43:55.118616Z", + "shell.execute_reply": "2026-05-18T16:43:55.117697Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CUDA available: True\n", + "x: torch.Size([2, 4096, 768]), dtype=torch.bfloat16\n", + "weight: torch.Size([4, 768]), dtype=torch.bfloat16\n", + "bias: torch.Size([768]), dtype=torch.bfloat16\n" + ] + } + ], + "source": [ + "batch = 2\n", + "dim = 768\n", + "seq_len = 4096\n", + "kernel_size = 4\n", + "dtype = torch.bfloat16\n", + "\n", + "has_cuda = torch.cuda.is_available()\n", + "\n", + "torch.manual_seed(42)\n", + "\n", + "x = torch.randn(batch, seq_len, dim, dtype=dtype)\n", + "weight = torch.randn(kernel_size, dim, dtype=dtype)\n", + "bias = torch.randn(dim, dtype=dtype)\n", + "\n", + "print(f\"CUDA available: {has_cuda}\")\n", + "print(f\"x: {x.shape}, dtype={x.dtype}\")\n", + "print(f\"weight: {weight.shape}, dtype={weight.dtype}\")\n", + "print(f\"bias: {bias.shape}, dtype={bias.dtype}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute Reference Output" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:43:55.120987Z", + "iopub.status.busy": "2026-05-18T16:43:55.120866Z", + "iopub.status.idle": "2026-05-18T16:43:55.134348Z", + "shell.execute_reply": "2026-05-18T16:43:55.133844Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "y_ref: torch.Size([2, 4096, 768]), dtype=torch.bfloat16\n" + ] + } + ], + "source": [ + "y_ref = causal_conv1d_nwh_ref(x, weight, bias, activation=\"silu\")\n", + "print(f\"y_ref: {y_ref.shape}, dtype={y_ref.dtype}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute cuDNN Output" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:43:55.136108Z", + "iopub.status.busy": "2026-05-18T16:43:55.135985Z", + "iopub.status.idle": "2026-05-18T16:44:02.070473Z", + "shell.execute_reply": "2026-05-18T16:44:02.069527Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "y_cudnn: torch.Size([2, 4096, 768]), dtype=torch.bfloat16\n" + ] + } + ], + "source": [ + "if has_cuda:\n", + " x_gpu = x.cuda()\n", + " weight_gpu = weight.cuda()\n", + " bias_gpu = bias.cuda()\n", + " y_cudnn = cudnn.ops.causal_conv1d_nwh(x_gpu, weight_gpu, bias=bias_gpu, activation=\"silu\")\n", + " print(f\"y_cudnn: {y_cudnn.shape}, dtype={y_cudnn.dtype}\")\n", + "else:\n", + " print(\"Skipping cuDNN forward (no CUDA device). Run on a GPU machine to test cuDNN.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Verify Correctness" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:44:02.072939Z", + "iopub.status.busy": "2026-05-18T16:44:02.072758Z", + "iopub.status.idle": "2026-05-18T16:44:02.102820Z", + "shell.execute_reply": "2026-05-18T16:44:02.102285Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Max absolute difference: 6.250000e-02\n", + "Max relative difference: 3.278166e-02\n", + "PASSED: cuDNN causal_conv1d NWH forward matches reference.\n" + ] + } + ], + "source": [ + "if has_cuda:\n", + " y_cudnn_cpu = y_cudnn.cpu()\n", + " max_abs_diff = (y_cudnn_cpu.float() - y_ref.float()).abs().max().item()\n", + " max_rel_diff = ((y_cudnn_cpu.float() - y_ref.float()).abs() / (y_ref.float().abs() + 1e-6)).max().item()\n", + "\n", + " print(f\"Max absolute difference: {max_abs_diff:.6e}\")\n", + " print(f\"Max relative difference: {max_rel_diff:.6e}\")\n", + "\n", + " atol = 1e-2\n", + " rtol = 1e-2\n", + "\n", + " assert torch.allclose(y_cudnn_cpu.float(), y_ref.float(), atol=atol, rtol=rtol), \\\n", + " f\"FAILED: max_abs={max_abs_diff}, max_rel={max_rel_diff}\"\n", + "\n", + " print(\"PASSED: cuDNN causal_conv1d NWH forward matches reference.\")\n", + "else:\n", + " print(\"Skipping verification (no CUDA device).\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Test with Different Data Types and Kernel Sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:44:02.105008Z", + "iopub.status.busy": "2026-05-18T16:44:02.104888Z", + "iopub.status.idle": "2026-05-18T16:44:25.783177Z", + "shell.execute_reply": "2026-05-18T16:44:25.782360Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dtype=torch.float32 K= 4 act=silu max_abs_diff=1.9073e-06\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dtype=torch.float16 K= 4 act=silu max_abs_diff=7.8125e-03\n", + "[PASS] dtype=torch.bfloat16 K= 4 act=silu max_abs_diff=6.2500e-02\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dtype=torch.bfloat16 K= 3 act=silu max_abs_diff=6.2500e-02\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dtype=torch.float16 K= 7 act=silu max_abs_diff=7.8125e-03\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dtype=torch.bfloat16 K= 4 act=identity max_abs_diff=2.4414e-04\n", + "\n", + "All tests passed!\n" + ] + } + ], + "source": [ + "tolerances = {\n", + " torch.float32: (5e-6, 5e-6),\n", + " torch.float16: (5e-3, 5e-3),\n", + " torch.bfloat16: (1e-2, 1e-2),\n", + "}\n", + "\n", + "test_configs = [\n", + " {\"dtype\": torch.float32, \"kernel_size\": 4, \"activation\": \"silu\"},\n", + " {\"dtype\": torch.float16, \"kernel_size\": 4, \"activation\": \"silu\"},\n", + " {\"dtype\": torch.bfloat16, \"kernel_size\": 4, \"activation\": \"silu\"},\n", + " {\"dtype\": torch.bfloat16, \"kernel_size\": 3, \"activation\": \"silu\"},\n", + " {\"dtype\": torch.float16, \"kernel_size\": 7, \"activation\": \"silu\"},\n", + " {\"dtype\": torch.bfloat16, \"kernel_size\": 4, \"activation\": \"identity\"},\n", + "]\n", + "\n", + "batch, dim, seq_len = 2, 256, 1024\n", + "\n", + "for cfg in test_configs:\n", + " dt = cfg[\"dtype\"]\n", + " ks = cfg[\"kernel_size\"]\n", + " act = cfg[\"activation\"]\n", + " atol, rtol = tolerances[dt]\n", + "\n", + " x = torch.randn(batch, seq_len, dim, dtype=dt)\n", + " w = torch.randn(ks, dim, dtype=dt)\n", + " b = torch.randn(dim, dtype=dt)\n", + "\n", + " y_ref = causal_conv1d_nwh_ref(x, w, b, activation=act)\n", + "\n", + " if has_cuda:\n", + " y_cudnn = cudnn.ops.causal_conv1d_nwh(x.cuda(), w.cuda(), bias=b.cuda(), activation=act)\n", + " max_abs = (y_cudnn.cpu().float() - y_ref.float()).abs().max().item()\n", + " passed = torch.allclose(y_cudnn.cpu().float(), y_ref.float(), atol=atol, rtol=rtol)\n", + " status = \"PASS\" if passed else \"FAIL\"\n", + " print(f\"[{status}] dtype={str(dt):15s} K={ks:3d} act={act:10s} max_abs_diff={max_abs:.4e}\")\n", + " assert passed, f\"Test failed for {cfg}\"\n", + " else:\n", + " print(f\"[REF ] dtype={str(dt):15s} K={ks:3d} act={act:10s} y_ref shape={y_ref.shape}\")\n", + "\n", + "print(\"\\nAll tests passed!\" if has_cuda else \"\\nAll reference tests completed (CPU only).\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## torch.compile Support\n", + "\n", + "`cudnn.ops.causal_conv1d_nwh` is registered as a PyTorch custom operator, so it works seamlessly with `torch.compile`." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:44:25.785642Z", + "iopub.status.busy": "2026-05-18T16:44:25.785523Z", + "iopub.status.idle": "2026-05-18T16:44:29.796859Z", + "shell.execute_reply": "2026-05-18T16:44:29.796178Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.compile matches eager: max_abs_diff=0.0000e+00\n", + "y_compiled: torch.Size([2, 1024, 256]), dtype=torch.bfloat16\n", + "PASSED: torch.compile produces identical output to eager mode.\n" + ] + } + ], + "source": [ + "if has_cuda:\n", + " B, D, L, K = 2, 256, 1024, 4\n", + " x_c = torch.randn(B, L, D, dtype=torch.bfloat16, device=\"cuda\")\n", + " w_c = torch.randn(K, D, dtype=torch.bfloat16, device=\"cuda\")\n", + " b_c = torch.randn(D, dtype=torch.bfloat16, device=\"cuda\")\n", + "\n", + " @torch.compile\n", + " def compiled_causal_conv1d_nwh(x, w, b):\n", + " return cudnn.ops.causal_conv1d_nwh(x, w, bias=b, activation=\"silu\")\n", + "\n", + " y_eager = cudnn.ops.causal_conv1d_nwh(x_c, w_c, bias=b_c, activation=\"silu\")\n", + " y_compiled = compiled_causal_conv1d_nwh(x_c, w_c, b_c)\n", + "\n", + " max_abs = (y_compiled.float() - y_eager.float()).abs().max().item()\n", + " print(f\"torch.compile matches eager: max_abs_diff={max_abs:.4e}\")\n", + " assert max_abs == 0.0, f\"torch.compile output differs from eager: {max_abs}\"\n", + " print(f\"y_compiled: {y_compiled.shape}, dtype={y_compiled.dtype}\")\n", + " print(\"PASSED: torch.compile produces identical output to eager mode.\")\n", + "else:\n", + " print(\"Skipping torch.compile test (no CUDA device).\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/samples/python/63_causal_conv1d_nwh_backward.ipynb b/samples/python/63_causal_conv1d_nwh_backward.ipynb new file mode 100644 index 0000000000..34936db8b4 --- /dev/null +++ b/samples/python/63_causal_conv1d_nwh_backward.ipynb @@ -0,0 +1,560 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Causal Conv1D NWH Backward Operation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook shows how to compute gradients for the causal depthwise 1D convolution using cuDNN with NWH (batch, seqLen, dim) layout \u2014 \"NWH\".\n", + "\n", + "Given the forward operation:\n", + "\n", + "$$y = \\text{Activation}(\\text{CausalConv1D}(x, w) + b)$$\n", + "\n", + "where Activation is Identity or SiLU, this notebook computes:\n", + "\n", + "- $dx$ \u2014 gradient w.r.t. input $x$\n", + "- $dw$ \u2014 gradient w.r.t. weight $w$\n", + "- $db$ \u2014 gradient w.r.t. bias $b$\n", + "\n", + "The `cudnn.ops.causal_conv1d_nwh` API integrates with `torch.autograd`, so backward is handled automatically via `.backward()`. We compare the cuDNN autograd result against a PyTorch reference." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites and Setup\n", + "This notebook requires an NVIDIA GPU (Hopper or later recommended) and cuDNN 9.24.0 or later.\n", + "\n", + "**Environment setup** \u2014 make sure the cuDNN runtime library and the `cudnn` Python package are discoverable before launching the notebook:\n", + "\n", + "- **Option A \u2013 pip install:**\n", + " ```bash\n", + " pip install nvidia-cudnn-frontend\n", + " ```\n", + "- **Option B \u2013 set paths manually:**\n", + " ```bash\n", + " export LD_LIBRARY_PATH=/path/to/cudnn/lib:${LD_LIBRARY_PATH}\n", + " export PYTHONPATH=/path/to/cudnn_frontend/build_python:${PYTHONPATH}\n", + " ```\n", + "\n", + "Adjust the paths above to match your local build or installation directory." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:44:37.158508Z", + "iopub.status.busy": "2026-05-18T16:44:37.158239Z", + "iopub.status.idle": "2026-05-18T16:44:37.162531Z", + "shell.execute_reply": "2026-05-18T16:44:37.161574Z" + } + }, + "outputs": [], + "source": [ + "# !nvidia-smi" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:44:37.164931Z", + "iopub.status.busy": "2026-05-18T16:44:37.164683Z", + "iopub.status.idle": "2026-05-18T16:44:38.723225Z", + "shell.execute_reply": "2026-05-18T16:44:38.722262Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cuDNN backend version: 92300\n" + ] + } + ], + "source": [ + "import torch\n", + "import torch.nn.functional as F\n", + "import cudnn\n", + "\n", + "print(\"cuDNN backend version:\", cudnn.backend_version())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Reference Implementation\n", + "\n", + "A PyTorch reference for causal depthwise conv1d forward (NWH layout), used with autograd to compute reference gradients." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:44:38.726238Z", + "iopub.status.busy": "2026-05-18T16:44:38.726114Z", + "iopub.status.idle": "2026-05-18T16:44:38.729934Z", + "shell.execute_reply": "2026-05-18T16:44:38.729277Z" + } + }, + "outputs": [], + "source": [ + "def causal_conv1d_nwh_ref(x, weight, bias, activation=\"silu\"):\n", + " \"\"\"Reference: causal depthwise conv1d for NWH layout.\n", + "\n", + " Args:\n", + " x: (batch, seq_len, dim)\n", + " weight: (kernel_size, dim)\n", + " bias: (dim,)\n", + " Returns:\n", + " y: (batch, seq_len, dim)\n", + " \"\"\"\n", + " batch, seq_len, dim = x.shape\n", + " kernel_size = weight.shape[0]\n", + "\n", + " x_nhw = x.transpose(1, 2)\n", + " w_nhw = weight.T.unsqueeze(1)\n", + "\n", + " x_padded = F.pad(x_nhw, (kernel_size - 1, 0))\n", + " y = F.conv1d(x_padded, w_nhw, bias=bias, groups=dim)\n", + "\n", + " if activation == \"silu\":\n", + " y = F.silu(y)\n", + "\n", + " return y.transpose(1, 2)\n", + "\n", + "\n", + "def causal_conv1d_nwh_bwd_ref(x, weight, bias, dy, activation=\"silu\"):\n", + " \"\"\"Reference backward via PyTorch autograd.\n", + "\n", + " Returns:\n", + " dx, dweight, dbias (all float32 for comparison)\n", + " \"\"\"\n", + " x = x.float().detach().requires_grad_(True)\n", + " weight = weight.float().detach().requires_grad_(True)\n", + " bias = bias.float().detach().requires_grad_(True)\n", + " dy = dy.float().detach()\n", + "\n", + " y = causal_conv1d_nwh_ref(x, weight, bias, activation=activation)\n", + " y.backward(dy)\n", + "\n", + " return x.grad, weight.grad, bias.grad" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Tensors" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:44:38.732289Z", + "iopub.status.busy": "2026-05-18T16:44:38.732179Z", + "iopub.status.idle": "2026-05-18T16:44:38.998568Z", + "shell.execute_reply": "2026-05-18T16:44:38.997743Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CUDA available: True\n", + "x: torch.Size([2, 4096, 768]), dtype=torch.bfloat16\n", + "weight: torch.Size([4, 768]), dtype=torch.bfloat16\n", + "bias: torch.Size([768]), dtype=torch.bfloat16\n", + "dy: torch.Size([2, 4096, 768]), dtype=torch.bfloat16\n" + ] + } + ], + "source": [ + "batch = 2\n", + "dim = 768\n", + "seq_len = 4096\n", + "kernel_size = 4\n", + "dtype = torch.bfloat16\n", + "\n", + "has_cuda = torch.cuda.is_available()\n", + "\n", + "torch.manual_seed(42)\n", + "\n", + "x = torch.randn(batch, seq_len, dim, dtype=dtype)\n", + "weight = torch.randn(kernel_size, dim, dtype=dtype)\n", + "bias = torch.randn(dim, dtype=dtype)\n", + "dy = torch.randn(batch, seq_len, dim, dtype=dtype)\n", + "\n", + "print(f\"CUDA available: {has_cuda}\")\n", + "print(f\"x: {x.shape}, dtype={x.dtype}\")\n", + "print(f\"weight: {weight.shape}, dtype={weight.dtype}\")\n", + "print(f\"bias: {bias.shape}, dtype={bias.dtype}\")\n", + "print(f\"dy: {dy.shape}, dtype={dy.dtype}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute Reference Gradients (CPU, via autograd)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:44:39.001238Z", + "iopub.status.busy": "2026-05-18T16:44:39.001120Z", + "iopub.status.idle": "2026-05-18T16:44:39.609466Z", + "shell.execute_reply": "2026-05-18T16:44:39.608916Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dx_ref: torch.Size([2, 4096, 768]), dtype=torch.float32\n", + "dw_ref: torch.Size([4, 768]), dtype=torch.float32\n", + "db_ref: torch.Size([768]), dtype=torch.float32\n" + ] + } + ], + "source": [ + "dx_ref, dw_ref, db_ref = causal_conv1d_nwh_bwd_ref(x, weight, bias, dy, activation=\"silu\")\n", + "print(f\"dx_ref: {dx_ref.shape}, dtype={dx_ref.dtype}\")\n", + "print(f\"dw_ref: {dw_ref.shape}, dtype={dw_ref.dtype}\")\n", + "print(f\"db_ref: {db_ref.shape}, dtype={db_ref.dtype}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute cuDNN Gradients (via autograd)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:44:39.611009Z", + "iopub.status.busy": "2026-05-18T16:44:39.610797Z", + "iopub.status.idle": "2026-05-18T16:44:51.721097Z", + "shell.execute_reply": "2026-05-18T16:44:51.720392Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dx_cudnn: torch.Size([2, 4096, 768]), dtype=torch.bfloat16\n", + "dw_cudnn: torch.Size([4, 768]), dtype=torch.bfloat16\n", + "db_cudnn: torch.Size([768]), dtype=torch.bfloat16\n" + ] + } + ], + "source": [ + "if has_cuda:\n", + " x_gpu = x.cuda().detach().requires_grad_(True)\n", + " w_gpu = weight.cuda().detach().requires_grad_(True)\n", + " b_gpu = bias.cuda().detach().requires_grad_(True)\n", + " dy_gpu = dy.cuda()\n", + "\n", + " y_cudnn = cudnn.ops.causal_conv1d_nwh(x_gpu, w_gpu, bias=b_gpu, activation=\"silu\")\n", + " y_cudnn.backward(dy_gpu)\n", + "\n", + " dx_cudnn = x_gpu.grad\n", + " dw_cudnn = w_gpu.grad\n", + " db_cudnn = b_gpu.grad\n", + " print(f\"dx_cudnn: {dx_cudnn.shape}, dtype={dx_cudnn.dtype}\")\n", + " print(f\"dw_cudnn: {dw_cudnn.shape}, dtype={dw_cudnn.dtype}\")\n", + " print(f\"db_cudnn: {db_cudnn.shape}, dtype={db_cudnn.dtype}\")\n", + "else:\n", + " print(\"Skipping cuDNN backward (no CUDA device).\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Verify Correctness" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:44:51.723947Z", + "iopub.status.busy": "2026-05-18T16:44:51.723825Z", + "iopub.status.idle": "2026-05-18T16:44:51.736168Z", + "shell.execute_reply": "2026-05-18T16:44:51.735287Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dx max_abs_diff=6.0602e-02\n", + "[PASS] dweight max_abs_diff=4.9779e-01\n", + "[PASS] dbias max_abs_diff=6.9855e-01\n", + "\n", + "PASSED: cuDNN causal_conv1d_nwh autograd backward matches reference.\n" + ] + } + ], + "source": [ + "if has_cuda:\n", + " atol = 1e-2\n", + " rtol = 1e-2\n", + "\n", + " for name, cudnn_grad, ref_grad in [\n", + " (\"dx\", dx_cudnn.cpu().float(), dx_ref),\n", + " (\"dweight\", dw_cudnn.cpu().float(), dw_ref),\n", + " (\"dbias\", db_cudnn.cpu().float(), db_ref),\n", + " ]:\n", + " max_abs = (cudnn_grad - ref_grad).abs().max().item()\n", + " passed = torch.allclose(cudnn_grad, ref_grad, atol=atol, rtol=rtol)\n", + " status = \"PASS\" if passed else \"FAIL\"\n", + " print(f\"[{status}] {name:8s} max_abs_diff={max_abs:.4e}\")\n", + " assert passed, f\"{name} verification failed: max_abs={max_abs}\"\n", + "\n", + " print(\"\\nPASSED: cuDNN causal_conv1d_nwh autograd backward matches reference.\")\n", + "else:\n", + " print(\"Skipping verification (no CUDA device).\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Test with Different Data Types and Kernel Sizes" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:44:51.738651Z", + "iopub.status.busy": "2026-05-18T16:44:51.738386Z", + "iopub.status.idle": "2026-05-18T16:45:42.084354Z", + "shell.execute_reply": "2026-05-18T16:45:42.083158Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dtype=torch.float32 K= 4 act=silu max_abs: dx=4.7684e-06 dw=3.0518e-05 db=5.7220e-05\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dtype=torch.float16 K= 4 act=silu max_abs: dx=3.9072e-03 dw=3.0807e-02 db=2.9617e-02\n", + "[PASS] dtype=torch.bfloat16 K= 4 act=silu max_abs: dx=3.1171e-02 dw=2.7429e-01 db=2.2432e-01\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dtype=torch.bfloat16 K= 3 act=silu max_abs: dx=3.1232e-02 dw=2.3302e-01 db=2.3697e-01\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dtype=torch.float16 K= 7 act=silu max_abs: dx=3.9034e-03 dw=3.1174e-02 db=2.5726e-02\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dtype=torch.bfloat16 K= 4 act=identity max_abs: dx=3.1250e-02 dw=4.8869e-01 db=2.4513e-01\n", + "\n", + "All tests passed!\n" + ] + } + ], + "source": [ + "tolerances = {\n", + " torch.float32: (5e-6, 5e-6),\n", + " torch.float16: (5e-3, 5e-3),\n", + " torch.bfloat16: (1e-2, 1e-2),\n", + "}\n", + "dw_db_atol, dw_db_rtol = 5e-3, 5e-3\n", + "\n", + "test_configs = [\n", + " {\"dtype\": torch.float32, \"kernel_size\": 4, \"activation\": \"silu\"},\n", + " {\"dtype\": torch.float16, \"kernel_size\": 4, \"activation\": \"silu\"},\n", + " {\"dtype\": torch.bfloat16, \"kernel_size\": 4, \"activation\": \"silu\"},\n", + " {\"dtype\": torch.bfloat16, \"kernel_size\": 3, \"activation\": \"silu\"},\n", + " {\"dtype\": torch.float16, \"kernel_size\": 7, \"activation\": \"silu\"},\n", + " {\"dtype\": torch.bfloat16, \"kernel_size\": 4, \"activation\": \"identity\"},\n", + "]\n", + "\n", + "batch, dim, seq_len = 2, 256, 1024\n", + "\n", + "for cfg in test_configs:\n", + " dt = cfg[\"dtype\"]\n", + " ks = cfg[\"kernel_size\"]\n", + " act = cfg[\"activation\"]\n", + " atol, rtol = tolerances[dt]\n", + "\n", + " x = torch.randn(batch, seq_len, dim, dtype=dt)\n", + " w = torch.randn(ks, dim, dtype=dt)\n", + " b = torch.randn(dim, dtype=dt)\n", + " dy = torch.randn(batch, seq_len, dim, dtype=dt)\n", + "\n", + " dx_ref, dw_ref, db_ref = causal_conv1d_nwh_bwd_ref(x, w, b, dy, activation=act)\n", + "\n", + " if has_cuda:\n", + " x_g = x.cuda().detach().requires_grad_(True)\n", + " w_g = w.cuda().detach().requires_grad_(True)\n", + " b_g = b.cuda().detach().requires_grad_(True)\n", + "\n", + " y_g = cudnn.ops.causal_conv1d_nwh(x_g, w_g, bias=b_g, activation=act)\n", + " y_g.backward(dy.cuda())\n", + "\n", + " dx_ok = torch.allclose(x_g.grad.cpu().float(), dx_ref, atol=atol, rtol=rtol)\n", + " dw_ok = torch.allclose(w_g.grad.cpu().float(), dw_ref, atol=max(atol, dw_db_atol), rtol=max(rtol, dw_db_rtol))\n", + " db_ok = torch.allclose(b_g.grad.cpu().float(), db_ref, atol=max(atol, dw_db_atol), rtol=max(rtol, dw_db_rtol))\n", + " passed = dx_ok and dw_ok and db_ok\n", + "\n", + " dx_abs = (x_g.grad.cpu().float() - dx_ref).abs().max().item()\n", + " dw_abs = (w_g.grad.cpu().float() - dw_ref).abs().max().item()\n", + " db_abs = (b_g.grad.cpu().float() - db_ref).abs().max().item()\n", + "\n", + " status = \"PASS\" if passed else \"FAIL\"\n", + " print(f\"[{status}] dtype={str(dt):15s} K={ks:3d} act={act:10s} \"\n", + " f\"max_abs: dx={dx_abs:.4e} dw={dw_abs:.4e} db={db_abs:.4e}\")\n", + " assert passed, f\"Test failed for {cfg}\"\n", + " else:\n", + " print(f\"[REF ] dtype={str(dt):15s} K={ks:3d} act={act:10s} \"\n", + " f\"dx={dx_ref.shape} dw={dw_ref.shape} db={db_ref.shape}\")\n", + "\n", + "print(\"\\nAll tests passed!\" if has_cuda else \"\\nAll reference tests completed (CPU only).\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## torch.compile Support\n", + "\n", + "`cudnn.ops.causal_conv1d_nwh` works with `torch.compile` for both forward and backward passes. The compiled graph captures the custom op nodes and the autograd backward." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:45:42.086956Z", + "iopub.status.busy": "2026-05-18T16:45:42.086824Z", + "iopub.status.idle": "2026-05-18T16:45:48.738717Z", + "shell.execute_reply": "2026-05-18T16:45:48.737716Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PASS] dx eager vs compiled max_abs_diff=0.0000e+00\n", + "[PASS] dweight eager vs compiled max_abs_diff=0.0000e+00\n", + "[PASS] dbias eager vs compiled max_abs_diff=0.0000e+00\n", + "\n", + "PASSED: torch.compile backward produces identical output to eager mode.\n" + ] + } + ], + "source": [ + "if has_cuda:\n", + " B, D, L, K = 2, 256, 1024, 4\n", + " act = \"silu\"\n", + "\n", + " def train_step(x, w, b, dy):\n", + " x = x.detach().requires_grad_(True)\n", + " w = w.detach().requires_grad_(True)\n", + " b = b.detach().requires_grad_(True)\n", + " y = cudnn.ops.causal_conv1d_nwh(x, w, bias=b, activation=act)\n", + " y.backward(dy)\n", + " return x.grad, w.grad, b.grad\n", + "\n", + " compiled_train_step = torch.compile(train_step)\n", + "\n", + " x_c = torch.randn(B, L, D, dtype=torch.bfloat16, device=\"cuda\")\n", + " w_c = torch.randn(K, D, dtype=torch.bfloat16, device=\"cuda\")\n", + " b_c = torch.randn(D, dtype=torch.bfloat16, device=\"cuda\")\n", + " dy_c = torch.randn(B, L, D, dtype=torch.bfloat16, device=\"cuda\")\n", + "\n", + " dx_eager, dw_eager, db_eager = train_step(x_c, w_c, b_c, dy_c)\n", + " dx_compiled, dw_compiled, db_compiled = compiled_train_step(x_c, w_c, b_c, dy_c)\n", + "\n", + " for name, eager, compiled in [\n", + " (\"dx\", dx_eager, dx_compiled),\n", + " (\"dweight\", dw_eager, dw_compiled),\n", + " (\"dbias\", db_eager, db_compiled),\n", + " ]:\n", + " max_abs = (eager.float() - compiled.float()).abs().max().item()\n", + " print(f\"[{'PASS' if max_abs == 0 else 'FAIL'}] {name:8s} eager vs compiled max_abs_diff={max_abs:.4e}\")\n", + " assert max_abs == 0.0, f\"{name}: torch.compile differs from eager\"\n", + "\n", + " print(\"\\nPASSED: torch.compile backward produces identical output to eager mode.\")\n", + "else:\n", + " print(\"Skipping torch.compile test (no CUDA device).\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/samples/python/64_b2b_causal_conv1d_forward.ipynb b/samples/python/64_b2b_causal_conv1d_forward.ipynb new file mode 100644 index 0000000000..80095c62fb --- /dev/null +++ b/samples/python/64_b2b_causal_conv1d_forward.ipynb @@ -0,0 +1,356 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2f156c39", + "metadata": {}, + "source": [ + "# Back-to-Back Causal Conv1D Forward" + ] + }, + { + "cell_type": "markdown", + "id": "77443d6b", + "metadata": {}, + "source": [ + "This notebook computes a **fused back-to-back causal conv1d** forward pass using cuDNN.\n", + "\n", + "The B2B kernel fuses three stages into a single kernel launch:\n", + "\n", + "1. **Projection conv** \u2014 depthwise causal conv1d across all `3*dim` channels\n", + "2. **Gating** \u2014 element-wise multiply of two projected channels, then mixer conv + skip connection\n", + "3. **Post-gating** \u2014 multiply the mixer output with the remaining projected channel\n", + "\n", + "```python\n", + "proj = causal_conv1d(x, weights_proj) # all 3*dim channels\n", + "gated = proj[:, 1::3, :] * proj[:, 2::3, :] # gate: ch1 * ch2\n", + "y = causal_conv1d(gated, weights_mixer) + skip_bias * gated # intermediate\n", + "y_gated = y * proj[:, 0::3, :] # final post-gated output\n", + "```\n", + "\n", + "**Tensor shapes (cuhyena convention):**\n", + "\n", + "| Tensor | Shape | Description |\n", + "|---|---|---|\n", + "| x | `(batch, 3*dim, seq_len)` | 3 input channels interleaved per dim |\n", + "| weights_proj | `(3*dim, K_proj)` | projection filters |\n", + "| weights_mixer | `(dim, K_mixer)` | mixer filter |\n", + "| skip_bias | `(dim,)` | skip-connection bias |\n", + "| y_gated | `(batch, dim, seq_len)` | final post-gated output (return value) |" + ] + }, + { + "cell_type": "markdown", + "id": "0f46a2f1", + "metadata": {}, + "source": [ + "## Prerequisites and Setup\n", + "This notebook requires an NVIDIA GPU (Hopper or later recommended) and cuDNN 9.24.0 or later.\n", + "\n", + "**Environment setup** \u2014 make sure the cuDNN runtime library and the `cudnn` Python package are discoverable before launching the notebook:\n", + "\n", + "- **Option A \u2013 pip install:**\n", + " ```bash\n", + " pip install nvidia-cudnn-frontend\n", + " ```\n", + "- **Option B \u2013 set paths manually:**\n", + " ```bash\n", + " export LD_LIBRARY_PATH=/path/to/cudnn/lib:${LD_LIBRARY_PATH}\n", + " export PYTHONPATH=/path/to/cudnn_frontend/build_python:${PYTHONPATH}\n", + " ```\n", + "\n", + "Adjust the paths above to match your local build or installation directory." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "a530764a", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:45:57.081137Z", + "iopub.status.busy": "2026-05-18T16:45:57.080886Z", + "iopub.status.idle": "2026-05-18T16:45:57.084639Z", + "shell.execute_reply": "2026-05-18T16:45:57.083869Z" + } + }, + "outputs": [], + "source": [ + "# !nvidia-smi" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "7fdf1335", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:45:57.087316Z", + "iopub.status.busy": "2026-05-18T16:45:57.087067Z", + "iopub.status.idle": "2026-05-18T16:45:57.090012Z", + "shell.execute_reply": "2026-05-18T16:45:57.089268Z" + } + }, + "outputs": [], + "source": [ + "# !pip install nvidia-cudnn-cu12\n", + "# !pip install nvidia-cudnn-frontend\n", + "# !pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128" + ] + }, + { + "cell_type": "markdown", + "id": "6b720317", + "metadata": {}, + "source": [ + "## Overview\n", + "\n", + "We will perform a fused B2B causal conv1d forward pass with:\n", + "\n", + "- batch size: 2\n", + "- dim (channels): 64\n", + "- sequence length: 512\n", + "- projection kernel size: 4\n", + "- mixer kernel size: 7\n", + "- data type: Float32\n", + "\n", + "We compare the cuDNN fused result against a decomposed PyTorch reference." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "eceb2e50", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:45:57.092413Z", + "iopub.status.busy": "2026-05-18T16:45:57.092181Z", + "iopub.status.idle": "2026-05-18T16:45:58.737671Z", + "shell.execute_reply": "2026-05-18T16:45:58.736681Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cuDNN backend version: 92300\n" + ] + } + ], + "source": [ + "import torch\n", + "import torch.nn.functional as F\n", + "import cudnn\n", + "\n", + "print(\"cuDNN backend version:\", cudnn.backend_version())" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "78460ee9", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:45:58.740350Z", + "iopub.status.busy": "2026-05-18T16:45:58.740227Z", + "iopub.status.idle": "2026-05-18T16:45:58.743526Z", + "shell.execute_reply": "2026-05-18T16:45:58.742950Z" + } + }, + "outputs": [], + "source": [ + "def causal_conv1d_ref(x, weight):\n", + " \"\"\"Depthwise causal conv1d.\n", + "\n", + " Args:\n", + " x: (batch, channels, seq_len)\n", + " weight: (channels, kernel_size)\n", + " \"\"\"\n", + " x_padded = F.pad(x, (weight.shape[1] - 1, 0))\n", + " return F.conv1d(x_padded, weight.unsqueeze(1), groups=x.shape[1])\n", + "\n", + "\n", + "def b2b_causal_conv1d_ref(x, weights_proj, weights_mixer, skip_bias):\n", + " \"\"\"Reference: decomposed B2B causal conv1d.\n", + "\n", + " Returns the post-gated final output ``y_gated``.\n", + "\n", + " Args:\n", + " x: (batch, 3*dim, seq_len)\n", + " weights_proj: (3*dim, K_proj)\n", + " weights_mixer: (dim, K_mixer)\n", + " skip_bias: (dim,)\n", + " \"\"\"\n", + " proj = causal_conv1d_ref(x, weights_proj)\n", + " gated = proj[:, 1::3, :] * proj[:, 2::3, :]\n", + " y = causal_conv1d_ref(gated, weights_mixer) + skip_bias[:, None] * gated\n", + " y_gated = y * proj[:, 0::3, :]\n", + " return y_gated" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "bd83adfe", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:45:58.746121Z", + "iopub.status.busy": "2026-05-18T16:45:58.746007Z", + "iopub.status.idle": "2026-05-18T16:45:58.974021Z", + "shell.execute_reply": "2026-05-18T16:45:58.973326Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CUDA available: True\n", + "x: torch.Size([2, 192, 512]), dtype=torch.float32\n", + "weights_proj: torch.Size([192, 4]), dtype=torch.float32\n", + "weights_mixer: torch.Size([64, 7]), dtype=torch.float32\n", + "skip_bias: torch.Size([64]), dtype=torch.float32\n" + ] + } + ], + "source": [ + "batch = 2\n", + "dim = 64\n", + "seq_len = 512\n", + "k_proj = 4\n", + "k_mixer = 7\n", + "dtype = torch.float32\n", + "\n", + "has_cuda = torch.cuda.is_available()\n", + "\n", + "torch.manual_seed(42)\n", + "\n", + "x = torch.randn(batch, 3 * dim, seq_len, dtype=dtype)\n", + "weights_proj = torch.randn(3 * dim, k_proj, dtype=dtype)\n", + "weights_mixer = torch.randn(dim, k_mixer, dtype=dtype)\n", + "skip_bias = torch.randn(dim, dtype=dtype)\n", + "\n", + "print(f\"CUDA available: {has_cuda}\")\n", + "print(f\"x: {x.shape}, dtype={x.dtype}\")\n", + "print(f\"weights_proj: {weights_proj.shape}, dtype={weights_proj.dtype}\")\n", + "print(f\"weights_mixer: {weights_mixer.shape}, dtype={weights_mixer.dtype}\")\n", + "print(f\"skip_bias: {skip_bias.shape}, dtype={skip_bias.dtype}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b42995eb", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:45:58.976270Z", + "iopub.status.busy": "2026-05-18T16:45:58.976155Z", + "iopub.status.idle": "2026-05-18T16:45:58.982557Z", + "shell.execute_reply": "2026-05-18T16:45:58.982079Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "y_gated_ref: torch.Size([2, 64, 512]), dtype=torch.float32\n" + ] + } + ], + "source": [ + "y_gated_ref = b2b_causal_conv1d_ref(x, weights_proj, weights_mixer, skip_bias)\n", + "print(f\"y_gated_ref: {y_gated_ref.shape}, dtype={y_gated_ref.dtype}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5e996675", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:45:58.983580Z", + "iopub.status.busy": "2026-05-18T16:45:58.983457Z", + "iopub.status.idle": "2026-05-18T16:46:05.455835Z", + "shell.execute_reply": "2026-05-18T16:46:05.455167Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "y_gated_cudnn: torch.Size([2, 64, 512]), dtype=torch.float32\n" + ] + } + ], + "source": [ + "if has_cuda:\n", + " y_gated_cudnn = cudnn.ops.b2b_causal_conv1d(\n", + " x.cuda(), weights_proj.cuda(), weights_mixer.cuda(), skip_bias.cuda(),\n", + " )\n", + " print(f\"y_gated_cudnn: {y_gated_cudnn.shape}, dtype={y_gated_cudnn.dtype}\")\n", + "else:\n", + " print(\"Skipping cuDNN forward (no CUDA device).\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "06182dbe", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:46:05.458359Z", + "iopub.status.busy": "2026-05-18T16:46:05.458175Z", + "iopub.status.idle": "2026-05-18T16:46:05.462260Z", + "shell.execute_reply": "2026-05-18T16:46:05.461851Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "y_gated max_abs_diff=1.5259e-04\n", + "\n", + "PASSED: cuDNN B2B causal_conv1d forward matches reference.\n" + ] + } + ], + "source": [ + "if has_cuda:\n", + " atol = 1e-3\n", + "\n", + " max_abs = (y_gated_cudnn.cpu().float() - y_gated_ref.float()).abs().max().item()\n", + " print(f\"y_gated max_abs_diff={max_abs:.4e}\")\n", + " assert max_abs < atol, f\"y_gated verification failed: max_abs={max_abs}\"\n", + "\n", + " print(\"\\nPASSED: cuDNN B2B causal_conv1d forward matches reference.\")\n", + "else:\n", + " print(\"Skipping verification (no CUDA device).\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/samples/python/65_b2b_causal_conv1d_backward.ipynb b/samples/python/65_b2b_causal_conv1d_backward.ipynb new file mode 100644 index 0000000000..622eac33c5 --- /dev/null +++ b/samples/python/65_b2b_causal_conv1d_backward.ipynb @@ -0,0 +1,387 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "e1ab1364", + "metadata": {}, + "source": [ + "# Back-to-Back Causal Conv1D Backward" + ] + }, + { + "cell_type": "markdown", + "id": "39055f70", + "metadata": {}, + "source": [ + "This notebook tests gradients for the **fused back-to-back causal conv1d** using cuDNN.\n", + "\n", + "The `cudnn.ops.b2b_causal_conv1d` API integrates with `torch.autograd`, so backward is handled\n", + "automatically via `.backward()`. The fused B2B backward kernel computes gradients flowing\n", + "through the post-gated final output `y_gated`:\n", + "\n", + "```python\n", + "y_gated = cudnn.ops.b2b_causal_conv1d(x, weights_proj, weights_mixer, skip_bias)\n", + "loss = y_gated.sum()\n", + "loss.backward()\n", + "```\n", + "\n", + "and compare the resulting gradients against a decomposed PyTorch reference:\n", + "\n", + "- $dx$ \u2014 gradient w.r.t. input, shape `(batch, 3*dim, seq_len)`\n", + "- $dw_{proj}$ \u2014 gradient w.r.t. projection weights, shape `(3*dim, K_{proj})`\n", + "- $dw_{mixer}$ \u2014 gradient w.r.t. mixer weights, shape `(dim, K_{mixer})`\n", + "- $db_{skip}$ \u2014 gradient w.r.t. skip bias, shape `(dim,)`\n", + "\n", + "All tensors use FP32 for numerical stability." + ] + }, + { + "cell_type": "markdown", + "id": "54522367", + "metadata": {}, + "source": [ + "## Prerequisites and Setup\n", + "This notebook requires an NVIDIA GPU (Hopper or later recommended) and cuDNN 9.24.0 or later.\n", + "\n", + "**Environment setup** \u2014 make sure the cuDNN runtime library and the `cudnn` Python package are discoverable before launching the notebook:\n", + "\n", + "- **Option A \u2013 pip install:**\n", + " ```bash\n", + " pip install nvidia-cudnn-frontend\n", + " ```\n", + "- **Option B \u2013 set paths manually:**\n", + " ```bash\n", + " export LD_LIBRARY_PATH=/path/to/cudnn/lib:${LD_LIBRARY_PATH}\n", + " export PYTHONPATH=/path/to/cudnn_frontend/build_python:${PYTHONPATH}\n", + " ```\n", + "\n", + "Adjust the paths above to match your local build or installation directory." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "910f2ebc", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:46:11.272990Z", + "iopub.status.busy": "2026-05-18T16:46:11.272719Z", + "iopub.status.idle": "2026-05-18T16:46:11.276955Z", + "shell.execute_reply": "2026-05-18T16:46:11.275881Z" + } + }, + "outputs": [], + "source": [ + "# !nvidia-smi" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "a6a24bf4", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:46:11.279531Z", + "iopub.status.busy": "2026-05-18T16:46:11.279278Z", + "iopub.status.idle": "2026-05-18T16:46:11.282476Z", + "shell.execute_reply": "2026-05-18T16:46:11.281610Z" + } + }, + "outputs": [], + "source": [ + "# !pip install nvidia-cudnn-cu12\n", + "# !pip install nvidia-cudnn-frontend\n", + "# !pip3 install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu128" + ] + }, + { + "cell_type": "markdown", + "id": "8fef0380", + "metadata": {}, + "source": [ + "## Overview\n", + "\n", + "We will test backward gradients for a B2B causal conv1d with:\n", + "\n", + "- batch size: 2\n", + "- dim (channels): 64\n", + "- sequence length: 512\n", + "- projection kernel size: 4\n", + "- mixer kernel size: 7\n", + "- data type: Float32\n", + "\n", + "We compare the cuDNN autograd result against a decomposed PyTorch reference." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "f792a17a", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:46:11.284761Z", + "iopub.status.busy": "2026-05-18T16:46:11.284518Z", + "iopub.status.idle": "2026-05-18T16:46:12.898210Z", + "shell.execute_reply": "2026-05-18T16:46:12.897211Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cuDNN backend version: 92300\n" + ] + } + ], + "source": [ + "import torch\n", + "import torch.nn.functional as F\n", + "import cudnn\n", + "\n", + "print(\"cuDNN backend version:\", cudnn.backend_version())" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "ae03df4d", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:46:12.901013Z", + "iopub.status.busy": "2026-05-18T16:46:12.900891Z", + "iopub.status.idle": "2026-05-18T16:46:12.904278Z", + "shell.execute_reply": "2026-05-18T16:46:12.903599Z" + } + }, + "outputs": [], + "source": [ + "def causal_conv1d_ref(x, weight):\n", + " \"\"\"Depthwise causal conv1d.\n", + "\n", + " Args:\n", + " x: (batch, channels, seq_len)\n", + " weight: (channels, kernel_size)\n", + " \"\"\"\n", + " x_padded = F.pad(x, (weight.shape[1] - 1, 0))\n", + " return F.conv1d(x_padded, weight.unsqueeze(1), groups=x.shape[1])\n", + "\n", + "\n", + "def b2b_causal_conv1d_ref(x, weights_proj, weights_mixer, skip_bias):\n", + " \"\"\"Reference: decomposed B2B causal conv1d.\n", + "\n", + " Returns the post-gated final output ``y_gated``.\n", + "\n", + " Args:\n", + " x: (batch, 3*dim, seq_len)\n", + " weights_proj: (3*dim, K_proj)\n", + " weights_mixer: (dim, K_mixer)\n", + " skip_bias: (dim,)\n", + " \"\"\"\n", + " proj = causal_conv1d_ref(x, weights_proj)\n", + " gated = proj[:, 1::3, :] * proj[:, 2::3, :]\n", + " y = causal_conv1d_ref(gated, weights_mixer) + skip_bias[:, None] * gated\n", + " y_gated = y * proj[:, 0::3, :]\n", + " return y_gated" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "47236289", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:46:12.906521Z", + "iopub.status.busy": "2026-05-18T16:46:12.906408Z", + "iopub.status.idle": "2026-05-18T16:46:13.110892Z", + "shell.execute_reply": "2026-05-18T16:46:13.109876Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CUDA available: True\n", + "x: torch.Size([2, 192, 512]), dtype=torch.float32\n", + "weights_proj: torch.Size([192, 4]), dtype=torch.float32\n", + "weights_mixer: torch.Size([64, 7]), dtype=torch.float32\n", + "skip_bias: torch.Size([64]), dtype=torch.float32\n" + ] + } + ], + "source": [ + "batch = 2\n", + "dim = 64\n", + "seq_len = 512\n", + "k_proj = 4\n", + "k_mixer = 7\n", + "dtype = torch.float32\n", + "\n", + "has_cuda = torch.cuda.is_available()\n", + "\n", + "torch.manual_seed(42)\n", + "\n", + "x = torch.randn(batch, 3 * dim, seq_len, dtype=dtype)\n", + "weights_proj = torch.randn(3 * dim, k_proj, dtype=dtype)\n", + "weights_mixer = torch.randn(dim, k_mixer, dtype=dtype)\n", + "skip_bias = torch.randn(dim, dtype=dtype)\n", + "\n", + "print(f\"CUDA available: {has_cuda}\")\n", + "print(f\"x: {x.shape}, dtype={x.dtype}\")\n", + "print(f\"weights_proj: {weights_proj.shape}, dtype={weights_proj.dtype}\")\n", + "print(f\"weights_mixer: {weights_mixer.shape}, dtype={weights_mixer.dtype}\")\n", + "print(f\"skip_bias: {skip_bias.shape}, dtype={skip_bias.dtype}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "3be98d0b", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:46:13.113024Z", + "iopub.status.busy": "2026-05-18T16:46:13.112905Z", + "iopub.status.idle": "2026-05-18T16:46:13.126676Z", + "shell.execute_reply": "2026-05-18T16:46:13.126222Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dx_ref: torch.Size([2, 192, 512])\n", + "dweights_proj_ref: torch.Size([192, 4])\n", + "dweights_mixer_ref: torch.Size([64, 7])\n", + "dskip_bias_ref: torch.Size([64])\n" + ] + } + ], + "source": [ + "x_ref = x.clone().requires_grad_(True)\n", + "wp_ref = weights_proj.clone().requires_grad_(True)\n", + "wm_ref = weights_mixer.clone().requires_grad_(True)\n", + "sb_ref = skip_bias.clone().requires_grad_(True)\n", + "\n", + "y_gated_ref = b2b_causal_conv1d_ref(x_ref, wp_ref, wm_ref, sb_ref)\n", + "loss_ref = y_gated_ref.sum()\n", + "loss_ref.backward()\n", + "\n", + "print(f\"dx_ref: {x_ref.grad.shape}\")\n", + "print(f\"dweights_proj_ref: {wp_ref.grad.shape}\")\n", + "print(f\"dweights_mixer_ref: {wm_ref.grad.shape}\")\n", + "print(f\"dskip_bias_ref: {sb_ref.grad.shape}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "65e32007", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:46:13.128372Z", + "iopub.status.busy": "2026-05-18T16:46:13.128191Z", + "iopub.status.idle": "2026-05-18T16:46:24.242459Z", + "shell.execute_reply": "2026-05-18T16:46:24.241428Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dx_cudnn: torch.Size([2, 192, 512])\n", + "dweights_proj_cudnn: torch.Size([192, 4])\n", + "dweights_mixer_cudnn: torch.Size([64, 7])\n", + "dskip_bias_cudnn: torch.Size([64])\n" + ] + } + ], + "source": [ + "if has_cuda:\n", + " x_gpu = x.cuda().requires_grad_(True)\n", + " wp_gpu = weights_proj.cuda().requires_grad_(True)\n", + " wm_gpu = weights_mixer.cuda().requires_grad_(True)\n", + " sb_gpu = skip_bias.cuda().requires_grad_(True)\n", + "\n", + " y_gated_cudnn = cudnn.ops.b2b_causal_conv1d(x_gpu, wp_gpu, wm_gpu, sb_gpu)\n", + " loss_cudnn = y_gated_cudnn.sum()\n", + " loss_cudnn.backward()\n", + "\n", + " print(f\"dx_cudnn: {x_gpu.grad.shape}\")\n", + " print(f\"dweights_proj_cudnn: {wp_gpu.grad.shape}\")\n", + " print(f\"dweights_mixer_cudnn: {wm_gpu.grad.shape}\")\n", + " print(f\"dskip_bias_cudnn: {sb_gpu.grad.shape}\")\n", + "else:\n", + " print(\"Skipping cuDNN backward (no CUDA device).\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "997f13d6", + "metadata": { + "execution": { + "iopub.execute_input": "2026-05-18T16:46:24.244819Z", + "iopub.status.busy": "2026-05-18T16:46:24.244491Z", + "iopub.status.idle": "2026-05-18T16:46:24.252775Z", + "shell.execute_reply": "2026-05-18T16:46:24.252082Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dx max_abs_diff=9.1553e-05\n", + "dweights_proj max_abs_diff=4.8828e-04\n", + "dweights_mixer max_abs_diff=2.4414e-04\n", + "dskip_bias max_abs_diff=1.2207e-04\n", + "\n", + "PASSED: cuDNN B2B causal_conv1d backward matches reference.\n" + ] + } + ], + "source": [ + "if has_cuda:\n", + " atol = 1e-3\n", + "\n", + " for name, cudnn_grad, ref_grad in [\n", + " (\"dx\", x_gpu.grad.cpu(), x_ref.grad),\n", + " (\"dweights_proj\", wp_gpu.grad.cpu(), wp_ref.grad),\n", + " (\"dweights_mixer\", wm_gpu.grad.cpu(), wm_ref.grad),\n", + " (\"dskip_bias\", sb_gpu.grad.cpu(), sb_ref.grad),\n", + " ]:\n", + " max_abs = (cudnn_grad - ref_grad).abs().max().item()\n", + " print(f\"{name:16s} max_abs_diff={max_abs:.4e}\")\n", + " assert max_abs < atol, f\"{name} verification failed: max_abs={max_abs}\"\n", + "\n", + " print(\"\\nPASSED: cuDNN B2B causal_conv1d backward matches reference.\")\n", + "else:\n", + " print(\"Skipping verification (no CUDA device).\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}