Skip to content
3 changes: 2 additions & 1 deletion docs/docs/sglang-diffusion/fused_kernels.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ These fusion families mount under both `quality="extra-high"` and
| `apply_group_norm_silu` | Triton | close | `GroupNorm + SiLU`, NCHW-contiguous |
| `group_norm_silu_4d`, `group_norm_silu_rows` | Triton | close | channels-last GroupNorm(+SiLU); what lets a VAE decoder run channels_last end to end with no `nchwToNhwc` transposes |
| `wan_rmsnorm_silu` | Triton | close | Wan VAE `channels_last_3d` RMSNorm + SiLU |
| `wan_norm_silu_post` | JIT CUDA | bit-exact, first-sight verified | Wan VAE FP32 divide, scale, affine, and SiLU after the unchanged native L2 reduction; dense NCDHW and `channels_last_3d` |

### adaLN modulation and gating

Expand Down Expand Up @@ -206,7 +207,7 @@ Kernels are written against a specific eager chain in a specific model, so cover
| Sana | LN+modulate, GLUMB bias+SiLU / bias+GLU, residual-gate add |
| SANA-Video | Packed QKV/KV; paired fp64 interleaved RoPE; LN+modulate, GLUMB bias+SiLU / bias+GLU, and residual-gate add during BCG; BF16-input linear attention at `quality=extra-high` or `quality=high` |
| Sana-WM | bidirectional gated delta-net, fused QK inverse-RMS |
| Wan | temb table slices; VAE cat+pad and DupUp3D add, `channels_last_3d` RMSNorm+SiLU |
| Wan / LongLive 2 VAE | temb table slices (Wan); VAE cat+pad and DupUp3D add; lossless FP32 normalization post-ops; full `channels_last_3d` RMSNorm+SiLU at `quality=extra-high` or `quality=high` |
| Cosmos3 / Krea2 / MiniMax-H3 | QK-norm + RoPE (Krea2 also CuTe-DSL norm+scale/shift; MiniMax-H3 also indexed modulation) |
| FLUX.2 VAE / HunyuanVAE / latent upsampler | GroupNorm + SiLU (channels-last two-pass for FLUX.2) |

Expand Down
115 changes: 115 additions & 0 deletions python/sglang/kernels/jit/csrc/elementwise/wan_norm_silu_post.cuh
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
// SPDX-License-Identifier: Apache-2.0
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>

#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/type.cuh>
#include <sgl_kernel/utils.cuh>
#include <sgl_kernel/vec.cuh>

#include <tvm/ffi/container/tensor.h>

#include <algorithm>
#include <cmath>
#include <cstdint>

namespace sglang {

/// \brief Fuse FP32 divide, scale, affine, and SiLU without changing aten's L2 reduction.
template <typename X, typename A, bool kChannelsLast>
__global__ void wan_norm_silu_post_kernel(
const X* __restrict__ input,
const float* __restrict__ denominator,
const A* __restrict__ gamma,
const A* __restrict__ bias,
float* __restrict__ output,
int64_t num_vecs,
int64_t channels,
int64_t spatial,
float scale,
bool has_bias) {
constexpr int kVec = 4;
const int64_t step = static_cast<int64_t>(blockDim.x) * gridDim.x;
for (int64_t vec = static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x; vec < num_vecs; vec += step) {
device::AlignedVector<X, kVec> x;
device::AlignedVector<float, kVec> d;
device::AlignedVector<A, kVec> g;
device::AlignedVector<A, kVec> b;
device::AlignedVector<float, kVec> result;
x.load(input, vec);
float shared_d = 0.f, shared_g = 0.f, shared_b = 0.f;
if constexpr (kChannelsLast) {
const int64_t channel = vec * kVec % channels;
shared_d = denominator[vec * kVec / channels];
g.load(gamma, channel / kVec);
if (has_bias) b.load(bias, channel / kVec);
} else {
const int64_t channel = vec * kVec / spatial % channels;
const int64_t batch = vec * kVec / (spatial * channels);
const int64_t pixel = vec * kVec % spatial;
d.load(denominator, (batch * spatial + pixel) / kVec);
shared_g = device::cast<float>(gamma[channel]);
if (has_bias) shared_b = device::cast<float>(bias[channel]);
}
#pragma unroll
for (int i = 0; i < kVec; ++i) {
const float denom = kChannelsLast ? shared_d : d[i];
const float weight = kChannelsLast ? device::cast<float>(g[i]) : shared_g;
const float offset = has_bias ? (kChannelsLast ? device::cast<float>(b[i]) : shared_b) : 0.f;
float y = __fdiv_rn(device::cast<float>(x[i]), denom);
y = __fmul_rn(y, scale);
y = __fmul_rn(y, weight);
y = __fadd_rn(y, offset);
result[i] = __fdiv_rn(y, __fadd_rn(1.f, expf(-y)));
}
result.store(output, vec);
}
}

/// \brief Launch normalization post-ops for dense NCDHW or channels-last storage.
/// \param denominator Native FP32 L2 norm, clamped to the normalization epsilon.
template <typename X, typename A, bool kChannelsLast>
void wan_norm_silu_post(
tvm::ffi::TensorView input,
tvm::ffi::TensorView denominator,
tvm::ffi::TensorView gamma,
tvm::ffi::TensorView bias,
tvm::ffi::TensorView output,
int64_t channels,
int64_t spatial,
double scale,
bool has_bias) {
using namespace host;
auto elements = SymbolicSize{"elements"};
auto pixels = SymbolicSize{"pixels"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLCUDA>();
TensorMatcher({elements}).with_dtype<X>().with_device(device_).verify(input);
TensorMatcher({pixels}).with_dtype<float>().with_device(device_).verify(denominator);
TensorMatcher({channels}).with_dtype<A>().with_device(device_).verify(gamma);
TensorMatcher({channels}).with_dtype<A>().with_device(device_).verify(bias);
TensorMatcher({elements}).with_dtype<float>().with_device(device_).verify(output);
const int64_t count = elements.unwrap();
CHECK_HOST(channels > 0 && channels % 4 == 0 && spatial > 0 && spatial % 4 == 0);
CHECK_HOST(count > 0 && count % (channels * spatial) == 0);
CHECK_HOST(pixels.unwrap() == count / channels);
constexpr int64_t threads = 256;
const auto kernel = wan_norm_silu_post_kernel<X, A, kChannelsLast>;
const auto blocks_per_sm = runtime::get_blocks_per_sm(kernel, threads);
const auto sms = runtime::get_sm_count(device_.unwrap().device_id);
const auto blocks = std::min<int64_t>(sms * blocks_per_sm, div_ceil(count / 4, threads));
LaunchKernel(blocks, threads, device_.unwrap())(
kernel,
static_cast<const X*>(input.data_ptr()),
static_cast<const float*>(denominator.data_ptr()),
static_cast<const A*>(gamma.data_ptr()),
static_cast<const A*>(bias.data_ptr()),
static_cast<float*>(output.data_ptr()),
count / 4,
channels,
spatial,
static_cast<float>(scale),
has_bias);
}

} // namespace sglang
9 changes: 9 additions & 0 deletions python/sglang/kernels/ops/diffusion/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,13 @@
_CUDA,
"Wan VAE channels_last_3d RMSNorm + SiLU.",
),
(
"diffusion.wan_norm_silu_post",
KernelBackend.JIT,
"norm.wan_norm_silu_post:wan_norm_silu_post",
_CUDA,
"Wan VAE FP32 normalization post-ops with the native denominator.",
),
(
"diffusion.rmsnorm_scale_shift",
KernelBackend.TRITON,
Expand Down Expand Up @@ -548,6 +555,8 @@
"try_fused_scale_residual_norm_scale_shift_fp8": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
"validate_scale_shift": "norm.scale_residual_norm_cutedsl",
"try_fused_scale_residual_norm_scale_shift_nvfp4": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
"can_use_wan_norm_silu_post": "norm.wan_norm_silu_post",
"wan_norm_silu_post": "norm.wan_norm_silu_post",
"can_use_wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton",
"wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton",
"can_use_qk_rmsnorm_native": "norm.zimage_qk_rmsnorm_triton",
Expand Down
99 changes: 99 additions & 0 deletions python/sglang/kernels/ops/diffusion/norm/wan_norm_silu_post.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
# SPDX-License-Identifier: Apache-2.0
"""FP32 Wan normalization post-ops with the native L2 denominator supplied."""

import torch

from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
from sglang.srt.utils.custom_op import register_custom_op


@cache_once
def _module(x_dtype, affine_dtype, channels_last):
args = make_cpp_args(x_dtype, affine_dtype, channels_last)
return load_jit(
"wan_norm_silu_post",
*args,
cuda_files=["elementwise/wan_norm_silu_post.cuh"],
extra_cuda_cflags=["--fmad=false"],
cuda_wrappers=[("run", f"wan_norm_silu_post<{args}>")],
)


def can_use_wan_norm_silu_post(x, denominator, gamma, bias=None):
if (
not x.is_cuda
or torch.version.hip is not None
or torch.is_grad_enabled()
or x.requires_grad
or x.ndim != 5
or x.numel() == 0
or x.dtype not in (torch.bfloat16, torch.float32)
or x.shape[1] % 4
or (x.shape[2] * x.shape[3] * x.shape[4]) % 4
or not (
x.is_contiguous() or x.is_contiguous(memory_format=torch.channels_last_3d)
)
or x.data_ptr() % 16
or denominator.dtype != torch.float32
or denominator.device != x.device
or denominator.shape != (x.shape[0], 1, *x.shape[2:])
or not denominator.is_contiguous()
or denominator.data_ptr() % 16
or gamma.dtype not in (torch.bfloat16, torch.float32)
or gamma.device != x.device
or gamma.numel() != x.shape[1]
or not gamma.is_contiguous()
or gamma.data_ptr() % 16
):
return False
return bias is None or (
bias.device == x.device
and bias.dtype == gamma.dtype
and bias.numel() == x.shape[1]
and bias.is_contiguous()
and bias.data_ptr() % 16 == 0
)


def _storage(x, channels_last):
return x.permute(0, 2, 3, 4, 1).view(-1) if channels_last else x.view(-1)


@register_custom_op(mutates_args=["out"])
def _wan_norm_silu_post(
x: torch.Tensor,
denominator: torch.Tensor,
gamma: torch.Tensor,
bias: torch.Tensor,
out: torch.Tensor,
scale: float,
has_bias: bool,
) -> None:
channels_last = not x.is_contiguous()
_module(x.dtype, gamma.dtype, channels_last).run(
_storage(x, channels_last),
denominator.view(-1),
gamma.view(-1),
bias.view(-1),
_storage(out, channels_last),
x.shape[1],
x.shape[2] * x.shape[3] * x.shape[4],
scale,
has_bias,
)


def wan_norm_silu_post(x, denominator, gamma, bias=None, *, scale=None):
if not can_use_wan_norm_silu_post(x, denominator, gamma, bias):
raise ValueError("unsupported Wan normalization post-op inputs")
out = torch.empty_like(x, dtype=torch.float32)
_wan_norm_silu_post(
x,
denominator,
gamma,
gamma if bias is None else bias,
out,
float(x.shape[1] ** 0.5 if scale is None else scale),
bias is not None,
)
return out
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@
the channels_last_3d layout. Wrappers are installed once at VAE load and
dispatch on a decode-scoped :class:`VaeFastPathGate`: ``quality="extra-high"``
and ``quality="high"`` run the fused kernel (not bitwise-identical to aten,
hence gated), while the ``"lossless"`` default runs the original module path
bit-for-bit. Install is all-or-nothing and fail-closed.
hence gated). The ``"lossless"`` path preserves aten's normalization reduction
and fuses its FP32 post-ops after first-sight exactness verification.
Install is all-or-nothing and fail-closed.
"""

import torch
Expand All @@ -24,9 +25,12 @@

try:
from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
can_use_nearest_upsample_nhwc,
can_use_wan_norm_silu_post,
can_use_wan_rmsnorm_silu,
nearest_upsample_nhwc,
wan_norm_silu_post,
wan_rmsnorm_silu,
)

Expand All @@ -36,10 +40,10 @@


class FusedWanRMSNormSiLU(nn.Module):
"""``WanRMS_norm`` + SiLU fused via the channels_last_3d Triton kernel;
falls back to the original op chain (bit-identical to norm + ``nn.SiLU``)
for unsupported inputs and whenever the gate is off. Steps aside under
``torch.compile``, where Inductor already fuses this chain."""
"""Quality-gated full fusion or lossless FP32 post-op fusion.

Unsupported inputs and ``torch.compile`` use the original op chain.
"""

def __init__(self, norm: nn.Module, gate: VaeFastPathGate) -> None:
super().__init__()
Expand All @@ -50,12 +54,58 @@ def __init__(self, norm: nn.Module, gate: VaeFastPathGate) -> None:
self.bias = norm.bias
self.scale = float(norm.scale)
self._sgl_gate = gate
self._post_gate = BitExactFusionGate(
"Wan norm+SiLU post-ops", per_signature=True
)

def forward(self, x: torch.Tensor) -> torch.Tensor:
if self._sgl_gate.enabled and not torch.compiler.is_compiling():
bias = self.bias if isinstance(self.bias, torch.Tensor) else None
if can_use_wan_rmsnorm_silu(x, self.gamma, bias):
return wan_rmsnorm_silu(x, self.gamma, bias, rms_scale=self.scale)
if (
not torch.compiler.is_compiling()
and not self._post_gate.disabled
and x.is_cuda
and torch.version.hip is None
and not torch.is_grad_enabled()
and x.ndim == 5
and (
x.dtype == torch.float32
or (x.dtype == torch.bfloat16 and torch.is_autocast_enabled("cuda"))
)
and (isinstance(self.bias, torch.Tensor) or self.bias == 0.0)
):
sig = (x.device, x.dtype, x.shape, x.stride(), self.gamma.dtype)
verified = self._post_gate.is_verified(sig)
if verified or not torch.cuda.is_current_stream_capturing():
# Keep this outside the custom op: CUDA autocast promotes the
# native norm to FP32, including for BF16 VAE activations.
denominator = x.norm(p=2, dim=1, keepdim=True).clamp_min(1e-12)
bias = self.bias if isinstance(self.bias, torch.Tensor) else None
if can_use_wan_norm_silu_post(x, denominator, self.gamma, bias):
try:
out = wan_norm_silu_post(
x, denominator, self.gamma, bias, scale=self.scale
)
if verified:
return out
ref = F.silu(
(x / denominator) * self.scale * self.gamma + self.bias
)
return self._post_gate.accept_or_fallback(
out,
ref,
sig=sig,
equal=lambda a, b: torch.equal(
a.view(torch.int32), b.view(torch.int32)
),
logger=logger,
)
except Exception as exc:
self._post_gate.on_exception(exc, logger=logger)
# Unsupported layouts/dtypes still reuse the native reduction.
return F.silu((x / denominator) * self.scale * self.gamma + self.bias)
# WanRMS_norm.forward (channel-first) + SiLU, same ops in the same
# order, so the off-path stays bit-identical.
return F.silu(F.normalize(x, dim=1) * self.scale * self.gamma + self.bias)
Expand Down
Loading
Loading