[PERF] Wan2.2 support adalayernorm fused op (#2585)

Signed-off-by: fan2956 <zhoufan53@huawei.com>
Co-authored-by: Canlin Guo <canlinguosdu@gmail.com>
This commit is contained in:
fan2956andCanlin Guo authored and GitHub committed 2026-04-15 14:02:41 +08:00
1 parent 1ad726f495
commit 2dff2d7c74
5 files changed
+45 -84

No files matched your search

@@ -744,9 +744,9 @@ class QwenImageTransformerBlock(nn.Module):
self.zero_cond_t = zero_cond_t
def _modulate(self, x, mod_params, index=None):
def _modulate(self, mod_params, index=None):
"""Apply modulation to input tensor"""
# x: b l d, shift: b d, scale: b d, gate: b d
# shift: b d, scale: b d, gate: b d
shift, scale, gate = mod_params.chunk(3, dim=-1)
if index is not None:
@@ -778,7 +778,7 @@ class QwenImageTransformerBlock(nn.Module):
scale_result = scale.unsqueeze(1)
gate_result = gate.unsqueeze(1)
return x * (1 + scale_result) + shift_result, gate_result
return scale_result, shift_result, gate_result
def forward(
self,
@@ -804,10 +804,12 @@ class QwenImageTransformerBlock(nn.Module):
txt_mod1, txt_mod2 = txt_mod_params.chunk(2, dim=-1) # Each [B, 3*dim]
# Process image stream - norm1 + modulation
img_modulated, img_gate1 = self.img_norm1(hidden_states, img_mod1, modulate_index)
img_scale1, img_shift1, img_gate1 = self._modulate(img_mod1, modulate_index)
img_modulated = self.img_norm1(hidden_states, img_scale1, img_shift1)
# Process text stream - norm1 + modulation
txt_modulated, txt_gate1 = self.txt_norm1(encoder_hidden_states, txt_mod1)
txt_scale1, txt_shift1, txt_gate1 = self._modulate(txt_mod1)
txt_modulated = self.txt_norm1(encoder_hidden_states, txt_scale1, txt_shift1)
# Use QwenAttnProcessor2_0 for joint attention computation
# This directly implements the DoubleStreamLayerMegatron logic:
@@ -832,13 +834,16 @@ class QwenImageTransformerBlock(nn.Module):
encoder_hidden_states = encoder_hidden_states + txt_gate1 * txt_attn_output
# Process image stream - norm2 + MLP
img_modulated2, img_gate2 = self.img_norm2(hidden_states, img_mod2, modulate_index)
img_scale2, img_shift2, img_gate2 = self._modulate(img_mod2, modulate_index)
img_modulated2 = self.img_norm2(hidden_states, img_scale2, img_shift2)
img_mlp_output = self.img_mlp(img_modulated2)
hidden_states = hidden_states + img_gate2 * img_mlp_output
# Process text stream - norm2 + MLP
txt_modulated2, txt_gate2 = self.txt_norm2(encoder_hidden_states, txt_mod2)
txt_scale2, txt_shift2, txt_gate2 = self._modulate(txt_mod2)
txt_modulated2 = self.txt_norm2(encoder_hidden_states, txt_scale2, txt_shift2)
txt_mlp_output = self.txt_mlp(txt_modulated2)
encoder_hidden_states = encoder_hidden_states + txt_gate2 * txt_mlp_output
@@ -29,6 +29,7 @@ from vllm_omni.diffusion.distributed.sp_plan import (
SequenceParallelOutput,
)
from vllm_omni.diffusion.forward_context import get_forward_context
from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNorm
from vllm_omni.platforms import current_omni_platform
logger = init_logger(__name__)
@@ -620,7 +621,7 @@ class WanTransformerBlock(nn.Module):
head_dim = dim // num_heads
# 1. Self-attention
self.norm1 = FP32LayerNorm(dim, eps, elementwise_affine=False)
self.norm1 = AdaLayerNorm(dim, elementwise_affine=False, eps=eps)
self.attn1 = WanSelfAttention(
dim=dim,
num_heads=num_heads,
@@ -640,7 +641,7 @@ class WanTransformerBlock(nn.Module):
# 3. Feed-forward
self.ffn = WanFeedForward(dim=dim, inner_dim=ffn_dim, dim_out=dim)
self.norm3 = FP32LayerNorm(dim, eps, elementwise_affine=False)
self.norm3 = AdaLayerNorm(dim, elementwise_affine=False, eps=eps)
# Scale-shift table for modulation
self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5)
@@ -656,7 +657,7 @@ class WanTransformerBlock(nn.Module):
if temb.ndim == 4:
# temb: batch_size, seq_len, 6, inner_dim (wan2.2 ti2v)
shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = (
self.scale_shift_table.unsqueeze(0) + temb.float()
self.scale_shift_table.unsqueeze(0) + temb
).chunk(6, dim=2)
shift_msa = shift_msa.squeeze(2)
scale_msa = scale_msa.squeeze(2)
@@ -667,25 +668,23 @@ class WanTransformerBlock(nn.Module):
else:
# temb: batch_size, 6, inner_dim (wan2.1/wan2.2 14B)
shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = (
self.scale_shift_table + temb.float()
self.scale_shift_table + temb
).chunk(6, dim=1)
# 1. Self-attention
norm_hidden_states = (self.norm1(hidden_states.float()) * (1 + scale_msa) + shift_msa).type_as(hidden_states)
norm_hidden_states = self.norm1(hidden_states, scale_msa, shift_msa).type_as(hidden_states)
attn_output = self.attn1(norm_hidden_states, rotary_emb, hidden_states_mask)
hidden_states = (hidden_states.float() + attn_output * gate_msa).type_as(hidden_states)
hidden_states = (hidden_states + attn_output * gate_msa).type_as(hidden_states)
# 2. Cross-attention
norm_hidden_states = self.norm2(hidden_states.float()).type_as(hidden_states)
norm_hidden_states = self.norm2(hidden_states).type_as(hidden_states)
attn_output = self.attn2(norm_hidden_states, encoder_hidden_states)
hidden_states = hidden_states + attn_output
# 3. Feed-forward
norm_hidden_states = (self.norm3(hidden_states.float()) * (1 + c_scale_msa) + c_shift_msa).type_as(
hidden_states
)
norm_hidden_states = self.norm3(hidden_states, c_scale_msa, c_shift_msa).type_as(hidden_states)
ff_output = self.ffn(norm_hidden_states)
hidden_states = (hidden_states.float() + ff_output.float() * c_gate_msa).type_as(hidden_states)
hidden_states = (hidden_states + ff_output * c_gate_msa).type_as(hidden_states)
return hidden_states
@@ -854,7 +853,7 @@ class WanTransformer3DModel(nn.Module):
)
# 4. Output norm & projection
self.norm_out = FP32LayerNorm(inner_dim, eps, elementwise_affine=False)
self.norm_out = AdaLayerNorm(inner_dim, elementwise_affine=False, eps=eps)
self.proj_out = nn.Linear(inner_dim, out_channels * math.prod(patch_size))
# SP helper modules
@@ -942,7 +941,7 @@ class WanTransformer3DModel(nn.Module):
shift = shift.unsqueeze(1)
scale = scale.unsqueeze(1)
hidden_states = (self.norm_out(hidden_states.float()) * (1 + scale) + shift).type_as(hidden_states)
hidden_states = self.norm_out(hidden_states, scale, shift).type_as(hidden_states)
hidden_states = self.proj_out(hidden_states)
hidden_states = hidden_states.reshape(
@@ -239,7 +239,7 @@ class WanVACETransformer3DModel(WanTransformer3DModel):
shift = shift.unsqueeze(1)
scale = scale.unsqueeze(1)
hidden_states = (self.norm_out(hidden_states.float()) * (1 + scale) + shift).type_as(hidden_states)
hidden_states = self.norm_out(hidden_states, scale, shift).type_as(hidden_states)
hidden_states = self.proj_out(hidden_states)
hidden_states = hidden_states.reshape(