Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 66 additions & 0 deletions vllm/_xpu_ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,46 @@ def _fused_add_gemma_rms_norm_impl(
torch.ops._C.fused_add_gemma_rms_norm(input, residual, weight, epsilon)


def _layer_norm_impl(
out: torch.Tensor,
input: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor,
epsilon: float,
) -> None:
torch.ops._C.layer_norm(out, input, weight, bias, epsilon)


def _fused_add_layer_norm_impl(
input: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor,
epsilon: float,
) -> None:
torch.ops._C.fused_add_layer_norm(input, residual, weight, bias, epsilon)


def _nemotron_layer_norm_impl(
out: torch.Tensor,
input: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor,
epsilon: float,
) -> None:
torch.ops._C.nemotron_layer_norm(out, input, weight, bias, epsilon)


def _fused_add_nemotron_layer_norm_impl(
input: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor,
epsilon: float,
) -> None:
torch.ops._C.fused_add_nemotron_layer_norm(input, residual, weight, bias, epsilon)


def _gdn_attention_core_xpu_impl(
core_attn_out: torch.Tensor,
z: torch.Tensor,
Expand Down Expand Up @@ -1246,6 +1286,32 @@ def register_ops_once() -> None:
mutates_args=["input", "residual"],
)

if hasattr(torch.ops._C, "layer_norm"):
direct_register_custom_op(
op_name="xpu_layer_norm",
op_func=_layer_norm_impl,
mutates_args=["out"],
)

direct_register_custom_op(
op_name="xpu_fused_add_layer_norm",
op_func=_fused_add_layer_norm_impl,
mutates_args=["input", "residual"],
)

if hasattr(torch.ops._C, "nemotron_layer_norm"):
direct_register_custom_op(
op_name="xpu_nemotron_layer_norm",
op_func=_nemotron_layer_norm_impl,
mutates_args=["out"],
)

direct_register_custom_op(
op_name="xpu_fused_add_nemotron_layer_norm",
op_func=_fused_add_nemotron_layer_norm_impl,
mutates_args=["input", "residual"],
)

direct_register_custom_op(
op_name="xpu_ops_deepseek_scaling_rope",
op_func=_xpu_ops_deepseek_scaling_rope_impl,
Expand Down
54 changes: 54 additions & 0 deletions vllm/model_executor/layers/layernorm.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,60 @@ def forward_xpu(
return out


@CustomOp.register("layer_norm")
class StandardLayerNorm(CustomOp):

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not just call it LayerNorm?

Suggested change
class StandardLayerNorm(CustomOp):
class LayerNorm(CustomOp):

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is already LayerNorm class being nn.Module at the end of the file, but I am open for alternatives for the name

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we port the other LayerNorm to be a custom op instead of adding a new one with a different name?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we could. Please, take a look and let me know if it looks good to you - ff208db

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"""Standard (mean-centered) LayerNorm.

Drop-in for a bare `nn.LayerNorm` that dispatches to a fused XPU SYCL
kernel (vllm-xpu-kernels `layer_norm`, PR #577); falls back to the native
implementation off XPU or when the op isn't in the installed
vllm-xpu-kernels package.
"""

def __init__(
self,
hidden_size: int,
eps: float = 1e-5,
elementwise_affine: bool = True,
bias: bool = True,
dtype: torch.dtype | None = None,
) -> None:
super().__init__()
self.normalized_shape = (hidden_size,)
self.eps = eps
weight_dtype = dtype or torch.get_default_dtype()
self.weight: nn.Parameter | None = None
self.bias: nn.Parameter | None = None
if elementwise_affine:
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=weight_dtype))
if bias:
self.bias = nn.Parameter(torch.zeros(hidden_size, dtype=weight_dtype))

def forward_native(self, x: torch.Tensor) -> torch.Tensor:
return F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps)

def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
return self.forward_native(x)

def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_layer_norm

if (
self.weight is None
or self.bias is None
or not hasattr(torch.ops._C, "layer_norm")
):
return self.forward_native(x)
# empty_like preserves x's strides, but the kernel requires a
# contiguous out (unlike x, which it can handle non-contiguous).
out = torch.empty(x.shape, device=x.device, dtype=x.dtype)
torch.ops.vllm.xpu_layer_norm(out, x, self.weight, self.bias, self.eps)
return out

def extra_repr(self) -> str:
return f"hidden_size={self.normalized_shape[0]}, eps={self.eps}"


# --8<-- [start:rms_norm_gated]
@CustomOp.register("rms_norm_gated")
class RMSNormGated(CustomOp):
Expand Down
7 changes: 4 additions & 3 deletions vllm/model_executor/models/gpt2.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
)
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import StandardLayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
Expand Down Expand Up @@ -156,11 +157,11 @@ def __init__(
hidden_size = config.hidden_size
inner_dim = config.n_inner if config.n_inner is not None else 4 * hidden_size

self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.ln_1 = StandardLayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.attn = GPT2Attention(
config, cache_config, quant_config, prefix=f"{prefix}.attn"
)
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.ln_2 = StandardLayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.mlp = GPT2MLP(inner_dim, config, quant_config, prefix=f"{prefix}.mlp")

def forward(
Expand Down Expand Up @@ -212,7 +213,7 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
lambda prefix: GPT2Block(config, cache_config, quant_config, prefix=prefix),
prefix=f"{prefix}.h",
)
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.ln_f = StandardLayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], config.n_embd
)
Expand Down
5 changes: 3 additions & 2 deletions vllm/model_executor/models/gpt_j.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import StandardLayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
Expand Down Expand Up @@ -163,7 +164,7 @@ def __init__(
):
super().__init__()
inner_dim = 4 * config.n_embd if config.n_inner is None else config.n_inner
self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
self.ln_1 = StandardLayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
self.attn = GPTJAttention(
config, cache_config, quant_config, prefix=f"{prefix}.attn"
)
Expand Down Expand Up @@ -206,7 +207,7 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
lambda prefix: GPTJBlock(config, cache_config, quant_config, prefix=prefix),
prefix=f"{prefix}.h",
)
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.ln_f = StandardLayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], config.n_embd
)
Expand Down
55 changes: 42 additions & 13 deletions vllm/model_executor/models/nemotron.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.linear import (
Expand Down Expand Up @@ -77,23 +78,22 @@ def _cast_if_autocast_enabled(device_type: str, *args):
)


class NemotronLayerNorm1P(nn.LayerNorm):
def __init__(
self,
normalized_shape: int | list[int] | torch.Size,
eps: float = 1e-5,
elementwise_affine: bool = True,
bias: bool = True,
device=None,
dtype=None,
):
super().__init__(normalized_shape, eps, elementwise_affine, bias, device, dtype)
@CustomOp.register("nemotron_layer_norm")
class NemotronLayerNorm1P(CustomOp):
"""LayerNorm variant used by Nemotron: `x * (1 + w)` instead of `x * w`."""

def forward(
def __init__(self, hidden_size: int, eps: float = 1e-5) -> None:
super().__init__()
self.normalized_shape = (hidden_size,)
self.eps = eps
self.weight = nn.Parameter(torch.zeros(hidden_size))
self.bias = nn.Parameter(torch.zeros(hidden_size))

def forward_native(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if residual is not None:
x = x + residual
residual = x
Expand All @@ -105,6 +105,35 @@ def forward(
x = torch.nn.functional.layer_norm(*args)
return x if residual is None else (x, residual)

def forward_cuda(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
return self.forward_native(x, residual)

def forward_xpu(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_nemotron_layer_norm

if not hasattr(torch.ops._C, "nemotron_layer_norm"):
return self.forward_native(x, residual)
if residual is not None:
if not hasattr(torch.ops._C, "fused_add_nemotron_layer_norm"):
return self.forward_native(x, residual)
torch.ops.vllm.xpu_fused_add_nemotron_layer_norm(
x, residual, self.weight, self.bias, self.eps
)
return x, residual
# empty_like preserves x's strides, but the kernel requires a
# contiguous out (unlike x, which it can handle non-contiguous).
out = torch.empty(x.shape, device=x.device, dtype=x.dtype)
torch.ops.vllm.xpu_nemotron_layer_norm(out, x, self.weight, self.bias, self.eps)
return out


class NemotronMLP(nn.Module):
def __init__(
Expand Down
7 changes: 4 additions & 3 deletions vllm/model_executor/models/opt.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import StandardLayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
Expand Down Expand Up @@ -143,7 +144,7 @@ def __init__(
)
self.do_layer_norm_before = config.do_layer_norm_before

self.self_attn_layer_norm = nn.LayerNorm(
self.self_attn_layer_norm = StandardLayerNorm(
self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine
)
self.fc1 = ColumnParallelLinear(
Expand All @@ -161,7 +162,7 @@ def __init__(
quant_config=quant_config,
prefix=f"{prefix}.fc2",
)
self.final_layer_norm = nn.LayerNorm(
self.final_layer_norm = StandardLayerNorm(
self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine
)

Expand Down Expand Up @@ -245,7 +246,7 @@ def __init__(
# before transformers v4.20.1
# see https://github.com/facebookresearch/metaseq/pull/164
if config.do_layer_norm_before and not config._remove_final_layer_norm:
self.final_layer_norm = nn.LayerNorm(
self.final_layer_norm = StandardLayerNorm(
config.hidden_size,
elementwise_affine=config.layer_norm_elementwise_affine,
)
Expand Down
16 changes: 11 additions & 5 deletions vllm/model_executor/models/siglip.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
MMEncoderAttention,
)
from vllm.model_executor.layers.conv import Conv2dLayer
from vllm.model_executor.layers.layernorm import StandardLayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
Expand Down Expand Up @@ -472,13 +473,13 @@ def __init__(
prefix=f"{prefix}.self_attn",
attn_cls=attn_cls,
)
self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
self.layer_norm1 = StandardLayerNorm(self.embed_dim, eps=config.layer_norm_eps)
self.mlp = SiglipMLP(
config,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
self.layer_norm2 = StandardLayerNorm(self.embed_dim, eps=config.layer_norm_eps)

def forward(
self,
Expand Down Expand Up @@ -578,7 +579,7 @@ def __init__(
attn_cls=EncoderOnlyAttention,
)

self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
self.final_layer_norm = StandardLayerNorm(embed_dim, eps=config.layer_norm_eps)
self.head = nn.Linear(embed_dim, config.projection_size)

def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
Expand Down Expand Up @@ -621,7 +622,9 @@ def __init__(
self.attention = torch.nn.MultiheadAttention(
config.hidden_size, config.num_attention_heads, batch_first=True
)
self.layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
self.layernorm = StandardLayerNorm(
config.hidden_size, eps=config.layer_norm_eps
)
self.mlp = SiglipMLP(
config=config,
quant_config=quant_config,
Expand Down Expand Up @@ -690,8 +693,11 @@ def __init__(
if require_post_norm is None:
require_post_norm = len(self.encoder.layers) == num_hidden_layers

self.post_layernorm: StandardLayerNorm | None
if require_post_norm:
self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
self.post_layernorm = StandardLayerNorm(
embed_dim, eps=config.layer_norm_eps
)
else:
self.post_layernorm = None

Expand Down
Loading
Loading