Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 66 additions & 0 deletions vllm/_xpu_ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,46 @@ def _fused_add_gemma_rms_norm_impl(
torch.ops._C.fused_add_gemma_rms_norm(input, residual, weight, epsilon)


def _layer_norm_impl(
out: torch.Tensor,
input: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor,
epsilon: float,
) -> None:
torch.ops._C.layer_norm(out, input, weight, bias, epsilon)


def _fused_add_layer_norm_impl(
input: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor,
epsilon: float,
) -> None:
torch.ops._C.fused_add_layer_norm(input, residual, weight, bias, epsilon)


def _nemotron_layer_norm_impl(
out: torch.Tensor,
input: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor,
epsilon: float,
) -> None:
torch.ops._C.nemotron_layer_norm(out, input, weight, bias, epsilon)


def _fused_add_nemotron_layer_norm_impl(
input: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor,
epsilon: float,
) -> None:
torch.ops._C.fused_add_nemotron_layer_norm(input, residual, weight, bias, epsilon)


def _gdn_attention_core_xpu_impl(
core_attn_out: torch.Tensor,
z: torch.Tensor,
Expand Down Expand Up @@ -1249,6 +1289,32 @@ def register_ops_once() -> None:
mutates_args=["input", "residual"],
)

if hasattr(torch.ops._C, "layer_norm"):
direct_register_custom_op(
op_name="xpu_layer_norm",
op_func=_layer_norm_impl,
mutates_args=["out"],
)

direct_register_custom_op(
op_name="xpu_fused_add_layer_norm",
op_func=_fused_add_layer_norm_impl,
mutates_args=["input", "residual"],
)

if hasattr(torch.ops._C, "nemotron_layer_norm"):
direct_register_custom_op(
op_name="xpu_nemotron_layer_norm",
op_func=_nemotron_layer_norm_impl,
mutates_args=["out"],
)

direct_register_custom_op(
op_name="xpu_fused_add_nemotron_layer_norm",
op_func=_fused_add_nemotron_layer_norm_impl,
mutates_args=["input", "residual"],
)

direct_register_custom_op(
op_name="xpu_ops_deepseek_scaling_rope",
op_func=_xpu_ops_deepseek_scaling_rope_impl,
Expand Down
89 changes: 73 additions & 16 deletions vllm/model_executor/layers/layernorm.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,79 @@ def forward_xpu(
return out


@CustomOp.register("layer_norm")
class LayerNorm(CustomOp):
"""Standard (mean-centered) LayerNorm.

Drop-in for a bare `nn.LayerNorm` that dispatches to a fused XPU kernel
when one is available, and to the native implementation otherwise.

Normalization runs in the wider of the input and parameter dtypes and the
result is cast back to the input dtype, so `dtype=torch.float32` gives an
fp32 reduction for a lower-precision input.
"""

def __init__(
self,
hidden_size: int,
eps: float = 1e-5,
elementwise_affine: bool = True,
bias: bool = True,
dtype: torch.dtype | None = None,
) -> None:
super().__init__()
self.normalized_shape = (hidden_size,)
self.eps = eps
weight_dtype = dtype or torch.get_default_dtype()
self.weight: nn.Parameter | None = None
self.bias: nn.Parameter | None = None
if elementwise_affine:
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=weight_dtype))
if bias:
self.bias = nn.Parameter(torch.zeros(hidden_size, dtype=weight_dtype))

def _compute_dtype(self, x: torch.Tensor) -> torch.dtype:
if self.weight is None:
return x.dtype
return torch.promote_types(x.dtype, self.weight.dtype)

def forward_native(self, x: torch.Tensor) -> torch.Tensor:
compute_dtype = self._compute_dtype(x)
if compute_dtype != x.dtype:
return F.layer_norm(
x.to(compute_dtype),
self.normalized_shape,
self.weight,
self.bias,
self.eps,
).type_as(x)
return F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps)

def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
return self.forward_native(x)

def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_layer_norm

if (
self.weight is None
or self.bias is None
or not hasattr(torch.ops._C, "layer_norm")
# The kernel reduces in the input dtype, so a widened reduction
# takes the native path.
or self._compute_dtype(x) != x.dtype
):
return self.forward_native(x)
# empty_like preserves x's strides, but the kernel requires a
# contiguous out (unlike x, which it can handle non-contiguous).
out = torch.empty(x.shape, device=x.device, dtype=x.dtype)
torch.ops.vllm.xpu_layer_norm(out, x, self.weight, self.bias, self.eps)
return out

def extra_repr(self) -> str:
return f"hidden_size={self.normalized_shape[0]}, eps={self.eps}"


# --8<-- [start:rms_norm_gated]
@CustomOp.register("rms_norm_gated")
class RMSNormGated(CustomOp):
Expand Down Expand Up @@ -333,19 +406,3 @@ def forward_xpu(
self, x: torch.Tensor, z: torch.Tensor | None = None
) -> torch.Tensor:
return self.forward_cuda(x, z)


class LayerNorm(nn.Module):
"""Layer Normalization."""

def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.dim = dim
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim, dtype=torch.float32))
self.bias = nn.Parameter(torch.zeros(dim, dtype=torch.float32))

def forward(self, x: torch.Tensor):
return F.layer_norm(
x.float(), (self.dim,), self.weight, self.bias, self.eps
).type_as(x)
2 changes: 1 addition & 1 deletion vllm/model_executor/models/deepseek_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -710,7 +710,7 @@ def __init__(
disable_tp=True,
prefix=f"{prefix}.wk_weights_proj",
)
self.k_norm = LayerNorm(self.head_dim, eps=1e-6)
self.k_norm = LayerNorm(self.head_dim, eps=1e-6, dtype=torch.float32)
self.softmax_scale = self.head_dim**-0.5

self.scale_fmt = "ue8m0"
Expand Down
7 changes: 4 additions & 3 deletions vllm/model_executor/models/gpt2.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
)
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
Expand Down Expand Up @@ -156,11 +157,11 @@ def __init__(
hidden_size = config.hidden_size
inner_dim = config.n_inner if config.n_inner is not None else 4 * hidden_size

self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.ln_1 = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.attn = GPT2Attention(
config, cache_config, quant_config, prefix=f"{prefix}.attn"
)
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.ln_2 = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.mlp = GPT2MLP(inner_dim, config, quant_config, prefix=f"{prefix}.mlp")

def forward(
Expand Down Expand Up @@ -212,7 +213,7 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
lambda prefix: GPT2Block(config, cache_config, quant_config, prefix=prefix),
prefix=f"{prefix}.h",
)
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.ln_f = LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], config.n_embd
)
Expand Down
5 changes: 3 additions & 2 deletions vllm/model_executor/models/gpt_j.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
Expand Down Expand Up @@ -163,7 +164,7 @@ def __init__(
):
super().__init__()
inner_dim = 4 * config.n_embd if config.n_inner is None else config.n_inner
self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
self.ln_1 = LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
self.attn = GPTJAttention(
config, cache_config, quant_config, prefix=f"{prefix}.attn"
)
Expand Down Expand Up @@ -206,7 +207,7 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
lambda prefix: GPTJBlock(config, cache_config, quant_config, prefix=prefix),
prefix=f"{prefix}.h",
)
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.ln_f = LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], config.n_embd
)
Expand Down
55 changes: 42 additions & 13 deletions vllm/model_executor/models/nemotron.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.linear import (
Expand Down Expand Up @@ -77,23 +78,22 @@ def _cast_if_autocast_enabled(device_type: str, *args):
)


class NemotronLayerNorm1P(nn.LayerNorm):
def __init__(
self,
normalized_shape: int | list[int] | torch.Size,
eps: float = 1e-5,
elementwise_affine: bool = True,
bias: bool = True,
device=None,
dtype=None,
):
super().__init__(normalized_shape, eps, elementwise_affine, bias, device, dtype)
@CustomOp.register("nemotron_layer_norm")
class NemotronLayerNorm1P(CustomOp):
"""LayerNorm variant used by Nemotron: `x * (1 + w)` instead of `x * w`."""

def forward(
def __init__(self, hidden_size: int, eps: float = 1e-5) -> None:
super().__init__()
self.normalized_shape = (hidden_size,)
self.eps = eps
self.weight = nn.Parameter(torch.zeros(hidden_size))
self.bias = nn.Parameter(torch.zeros(hidden_size))

def forward_native(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if residual is not None:
x = x + residual
residual = x
Expand All @@ -105,6 +105,35 @@ def forward(
x = torch.nn.functional.layer_norm(*args)
return x if residual is None else (x, residual)

def forward_cuda(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
return self.forward_native(x, residual)

def forward_xpu(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_nemotron_layer_norm

if not hasattr(torch.ops._C, "nemotron_layer_norm"):
return self.forward_native(x, residual)
if residual is not None:
if not hasattr(torch.ops._C, "fused_add_nemotron_layer_norm"):
return self.forward_native(x, residual)
torch.ops.vllm.xpu_fused_add_nemotron_layer_norm(
x, residual, self.weight, self.bias, self.eps
)
return x, residual
# empty_like preserves x's strides, but the kernel requires a
# contiguous out (unlike x, which it can handle non-contiguous).
out = torch.empty(x.shape, device=x.device, dtype=x.dtype)
torch.ops.vllm.xpu_nemotron_layer_norm(out, x, self.weight, self.bias, self.eps)
return out


class NemotronMLP(nn.Module):
def __init__(
Expand Down
8 changes: 5 additions & 3 deletions vllm/model_executor/models/opt.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
Expand Down Expand Up @@ -143,7 +144,7 @@ def __init__(
)
self.do_layer_norm_before = config.do_layer_norm_before

self.self_attn_layer_norm = nn.LayerNorm(
self.self_attn_layer_norm = LayerNorm(
self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine
)
self.fc1 = ColumnParallelLinear(
Expand All @@ -161,7 +162,7 @@ def __init__(
quant_config=quant_config,
prefix=f"{prefix}.fc2",
)
self.final_layer_norm = nn.LayerNorm(
self.final_layer_norm = LayerNorm(
self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine
)

Expand Down Expand Up @@ -246,8 +247,9 @@ def __init__(
# keep backward compatibility with checkpoints that have been fine-tuned
# before transformers v4.20.1
# see https://github.com/facebookresearch/metaseq/pull/164
self.final_layer_norm: LayerNorm | None
if config.do_layer_norm_before and not config._remove_final_layer_norm:
self.final_layer_norm = nn.LayerNorm(
self.final_layer_norm = LayerNorm(
config.hidden_size,
elementwise_affine=config.layer_norm_elementwise_affine,
)
Expand Down
Loading
Loading