[Diffusion] Fuse Qwen-Image-Edit select01 modulation for CUDA (#5921)

Signed-off-by: dongbo910220 <1275604947@qq.com>
This commit is contained in:
dongbo910220 authored and GitHub committed 2026-09-16 19:06:28 +08:00
1 parent 249e84d21b
commit 217a6af52c
5 files changed
+1255 -38

No files matched your search

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
@@ -49,6 +49,12 @@ from vllm_omni.diffusion.layers.fused_qk_norm_rope import (
_fused_cuda_supported,
fused_qk_norm_rope,
)
from vllm_omni.diffusion.layers.qwen_select01_modulation import (
can_use_qwen_select01_triton,
fused_layernorm_select01,
fused_residual_layernorm_select01,
select01_modulation_native,
)
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
logger = init_logger(__name__)
@@ -855,41 +861,11 @@ class QwenImageTransformerBlock(nn.Module):
self.zero_cond_t = zero_cond_t
def _modulate(self, mod_params, index=None):
def _modulate(self, mod_params):
"""Apply modulation to input tensor"""
# shift: b d, scale: b d, gate: b d
shift, scale, gate = mod_params.chunk(3, dim=-1)
if index is not None:
# Assuming mod_params batch dim is 2*actual_batch (chunked into 2 parts)
# So shift, scale, gate have shape [2*actual_batch, d]
actual_batch = shift.size(0) // 2
shift_0, shift_1 = shift[:actual_batch], shift[actual_batch:] # each: [actual_batch, d]
scale_0, scale_1 = scale[:actual_batch], scale[actual_batch:]
gate_0, gate_1 = gate[:actual_batch], gate[actual_batch:]
# index: [b, l] where b is actual batch size
# Expand to [b, l, 1] to match feature dimension
index_expanded = index.unsqueeze(-1) # [b, l, 1]
# Expand chunks to [b, 1, d] then broadcast to [b, l, d]
shift_0_exp = shift_0.unsqueeze(1) # [b, 1, d]
shift_1_exp = shift_1.unsqueeze(1) # [b, 1, d]
scale_0_exp = scale_0.unsqueeze(1)
scale_1_exp = scale_1.unsqueeze(1)
gate_0_exp = gate_0.unsqueeze(1)
gate_1_exp = gate_1.unsqueeze(1)
# Use torch.where to select based on index
shift_result = torch.where(index_expanded == 0, shift_0_exp, shift_1_exp)
scale_result = torch.where(index_expanded == 0, scale_0_exp, scale_1_exp)
gate_result = torch.where(index_expanded == 0, gate_0_exp, gate_1_exp)
else:
shift_result = shift.unsqueeze(1)
scale_result = scale.unsqueeze(1)
gate_result = gate.unsqueeze(1)
return scale_result, shift_result, gate_result
return scale.unsqueeze(1), shift.unsqueeze(1), gate.unsqueeze(1)
def forward(
self,
@@ -915,8 +891,22 @@ class QwenImageTransformerBlock(nn.Module):
txt_mod1, txt_mod2 = txt_mod_params.chunk(2, dim=-1) # Each [B, 3*dim]
# Process image stream - norm1 + modulation
img_scale1, img_shift1, img_gate1 = self._modulate(img_mod1, modulate_index)
img_modulated = self.img_norm1(hidden_states, img_scale1, img_shift1)
use_fused_select01 = modulate_index is not None and can_use_qwen_select01_triton(hidden_states)
if use_fused_select01:
img_modulated, img_gate1 = fused_layernorm_select01(
hidden_states,
img_mod1,
modulate_index,
self.img_norm1.eps,
self.img_norm1.layernorm.weight,
self.img_norm1.layernorm.bias,
)
elif modulate_index is not None:
img_scale1, img_shift1, img_gate1 = select01_modulation_native(img_mod1, modulate_index)
img_modulated = self.img_norm1(hidden_states, img_scale1, img_shift1)
else:
img_scale1, img_shift1, img_gate1 = self._modulate(img_mod1)
img_modulated = self.img_norm1(hidden_states, img_scale1, img_shift1)
# Process text stream - norm1 + modulation
txt_scale1, txt_shift1, txt_gate1 = self._modulate(txt_mod1)
@@ -941,12 +931,29 @@ class QwenImageTransformerBlock(nn.Module):
img_attn_output, txt_attn_output = attn_output
# Apply attention gates and add residual (like in Megatron)
hidden_states = hidden_states + img_gate1 * img_attn_output
hidden_states_before_attn = hidden_states
if not use_fused_select01:
hidden_states = hidden_states + img_gate1 * img_attn_output
encoder_hidden_states = encoder_hidden_states + txt_gate1 * txt_attn_output
# Process image stream - norm2 + MLP
img_scale2, img_shift2, img_gate2 = self._modulate(img_mod2, modulate_index)
img_modulated2 = self.img_norm2(hidden_states, img_scale2, img_shift2)
if use_fused_select01:
img_modulated2, hidden_states, img_gate2 = fused_residual_layernorm_select01(
img_attn_output,
hidden_states_before_attn,
img_gate1,
img_mod2,
modulate_index,
self.img_norm2.eps,
self.img_norm2.layernorm.weight,
self.img_norm2.layernorm.bias,
)
elif modulate_index is not None:
img_scale2, img_shift2, img_gate2 = select01_modulation_native(img_mod2, modulate_index)
img_modulated2 = self.img_norm2(hidden_states, img_scale2, img_shift2)
else:
img_scale2, img_shift2, img_gate2 = self._modulate(img_mod2)
img_modulated2 = self.img_norm2(hidden_states, img_scale2, img_shift2)
img_mlp_output = self.img_mlp(img_modulated2)
hidden_states = hidden_states + img_gate2 * img_mlp_output