[PERF] Wan2.2 support rmsnorm fused op (#2583)

Signed-off-by: fan2956 <zhoufan53@huawei.com>
Signed-off-by: gcanlin <canlinguosdu@gmail.com>
Co-authored-by: gcanlin <canlinguosdu@gmail.com>
This commit is contained in:
fan2956andgcanlin authored and GitHub committed 2026-04-16 03:21:34 -07:00
1 parent 4d816ff1de
commit f1cb4ebe4c
4 files changed
+585 -10

No files matched your search

+2 -1
View File
@@ -7,6 +7,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm_omni.diffusion.layers.custom_op import CustomOp
from vllm_omni.diffusion.layers.norm import LayerNorm
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
@@ -27,7 +28,7 @@ class AdaLayerNorm(CustomOp):
self.eps = eps
self.elementwise_affine = elementwise_affine
self.hidden_size = hidden_size
self.layernorm = nn.LayerNorm(self.hidden_size, elementwise_affine=self.elementwise_affine, eps=self.eps)
self.layernorm = LayerNorm(self.hidden_size, elementwise_affine=self.elementwise_affine, eps=self.eps)
def forward_cuda(
self,
+110
View File
@@ -0,0 +1,110 @@
from importlib.util import find_spec
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.logger import init_logger
from vllm_omni.diffusion.layers.custom_op import CustomOp
logger = init_logger(__name__)
_HAS_MINDIESD = find_spec("mindiesd") is not None
class LayerNorm(nn.LayerNorm, CustomOp):
"""
LayerNorm implementation that inherits from both ``nn.LayerNorm`` and ``CustomOp``.
NPU:
Uses ``mindiesd.fast_layernorm(self, x)`` when MindIE-SD is installed.
CUDA / HIP / XPU / native:
Falls back to FP32 nn.LayerNorm implementation.
"""
def __init__(self, dim: int, eps: float = 1e-6, elementwise_affine: bool = True):
super().__init__(normalized_shape=dim, eps=eps, elementwise_affine=elementwise_affine)
# CustomOp.__init__ cannot be called here because it would re-run
# nn.Module initialization and clear LayerNorm parameters.
self._forward_method = CustomOp.dispatch_forward(self)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self._forward_method(x)
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
return self.forward_native(x)
def forward_hip(self, x: torch.Tensor) -> torch.Tensor:
return self.forward_native(x)
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
return self.forward_native(x)
def forward_npu(self, x: torch.Tensor) -> torch.Tensor:
if _HAS_MINDIESD:
try:
from mindiesd import fast_layernorm
return fast_layernorm(self, x)
except ImportError as e:
logger.warning_once(
"mindiesd.fast_layernorm import failed, falling back to FP32 layer_norm: %s",
e,
)
return self.forward_native(x)
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
origin_dtype = x.dtype
return F.layer_norm(
x.float(),
self.normalized_shape,
self.weight.float() if self.weight is not None else None,
self.bias.float() if self.bias is not None else None,
self.eps,
).to(origin_dtype)
class RMSNorm(CustomOp):
def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
def forward_cuda(
self,
x: torch.Tensor,
) -> torch.Tensor:
return self.forward_native(x)
def forward_hip(
self,
x: torch.Tensor,
) -> torch.Tensor:
return self.forward_native(x)
def forward_npu(
self,
x: torch.Tensor,
) -> torch.Tensor:
import torch_npu
output = torch_npu.npu_rms_norm(x, gamma=self.weight, epsilon=self.variance_epsilon)[0]
return output
def forward_xpu(
self,
x: torch.Tensor,
) -> torch.Tensor:
return self.forward_native(x)
def forward_native(
self,
x: torch.Tensor,
) -> torch.Tensor:
input_dtype = x.dtype
x = x.to(torch.float32)
variance = x.pow(2).mean(-1, keepdim=True)
out = x * torch.rsqrt(variance + self.variance_epsilon)
out = self.weight.to(torch.float32) * out
return out.to(input_dtype)
@@ -11,7 +11,6 @@ import torch.nn.functional as F
from diffusers.models.attention import FeedForward
from diffusers.models.embeddings import PixArtAlphaTextProjection, TimestepEmbedding, Timesteps
from diffusers.models.modeling_outputs import Transformer2DModelOutput
from diffusers.models.normalization import FP32LayerNorm
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -30,6 +29,7 @@ from vllm_omni.diffusion.distributed.sp_plan import (
)
from vllm_omni.diffusion.forward_context import get_forward_context
from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNorm
from vllm_omni.diffusion.layers.norm import LayerNorm, RMSNorm
from vllm_omni.platforms import current_omni_platform
logger = init_logger(__name__)
@@ -236,9 +236,9 @@ class WanImageEmbedding(nn.Module):
def __init__(self, in_features: int, out_features: int, pos_embed_seq_len: int | None = None):
super().__init__()
self.norm1 = FP32LayerNorm(in_features)
self.norm1 = LayerNorm(in_features)
self.ff = FeedForward(in_features, out_features, mult=1, activation_fn="gelu")
self.norm2 = FP32LayerNorm(out_features)
self.norm2 = LayerNorm(out_features)
if pos_embed_seq_len is not None:
self.pos_embed = nn.Parameter(torch.zeros(1, pos_embed_seq_len, in_features))
else:
@@ -378,8 +378,12 @@ class WanSelfAttention(nn.Module):
self.tp_inner_dim = self.num_heads * head_dim
# QK normalization using vLLM's RMSNorm
self.norm_q = DistributedRMSNorm(self.tp_inner_dim, eps=eps)
self.norm_k = DistributedRMSNorm(self.tp_inner_dim, eps=eps)
if get_tensor_model_parallel_world_size() > 1:
self.norm_q = DistributedRMSNorm(self.tp_inner_dim, eps=eps)
self.norm_k = DistributedRMSNorm(self.tp_inner_dim, eps=eps)
else:
self.norm_q = RMSNorm(self.tp_inner_dim, eps=eps)
self.norm_k = RMSNorm(self.tp_inner_dim, eps=eps)
self.to_out = RowParallelLinear(
self.inner_dim,
@@ -498,8 +502,12 @@ class WanCrossAttention(nn.Module):
self.tp_inner_dim = self.num_heads * head_dim
# QK normalization
self.norm_q = DistributedRMSNorm(self.tp_inner_dim, eps=eps)
self.norm_k = DistributedRMSNorm(self.tp_inner_dim, eps=eps)
if get_tensor_model_parallel_world_size() > 1:
self.norm_q = DistributedRMSNorm(self.tp_inner_dim, eps=eps)
self.norm_k = DistributedRMSNorm(self.tp_inner_dim, eps=eps)
else:
self.norm_q = RMSNorm(self.tp_inner_dim, eps=eps)
self.norm_k = RMSNorm(self.tp_inner_dim, eps=eps)
# Optional added KV projections for I2V (image embeddings)
self.added_kv_proj_dim = added_kv_proj_dim
@@ -518,7 +526,10 @@ class WanCrossAttention(nn.Module):
gather_output=False,
return_bias=False,
)
self.norm_added_k = DistributedRMSNorm(self.tp_inner_dim, eps=eps)
if get_tensor_model_parallel_world_size() > 1:
self.norm_added_k = DistributedRMSNorm(self.tp_inner_dim, eps=eps)
else:
self.norm_added_k = RMSNorm(self.tp_inner_dim, eps=eps)
else:
self.add_k_proj = None
self.add_v_proj = None
@@ -637,7 +648,7 @@ class WanTransformerBlock(nn.Module):
eps=eps,
added_kv_proj_dim=added_kv_proj_dim,
)
self.norm2 = FP32LayerNorm(dim, eps, elementwise_affine=True) if cross_attn_norm else nn.Identity()
self.norm2 = LayerNorm(dim, eps, elementwise_affine=True) if cross_attn_norm else nn.Identity()
# 3. Feed-forward
self.ffn = WanFeedForward(dim=dim, inner_dim=ffn_dim, dim_out=dim)