[Bugfix] Fix generation artifacts of Qwen-Image-Edit-2511 and update pipeline DiT param parsing (#776)

Signed-off-by: samithuang <285365963@qq.com>
This commit is contained in:
Samit authored and GitHub committed 2026-01-15 11:18:28 +08:00
1 parent 314bb4e324
commit 2d5faf3634
10 files changed
+89 -23

No files matched your search

@@ -36,6 +36,7 @@ from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
QwenImageTransformer2DModel,
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs
from vllm_omni.model_executor.model_loader.weight_utils import (
download_weights_from_hf_specific,
)
@@ -274,7 +275,8 @@ class QwenImagePipeline(
self.vae = AutoencoderKLQwenImage.from_pretrained(model, subfolder="vae", local_files_only=local_files_only).to(
self.device
)
self.transformer = QwenImageTransformer2DModel(od_config=od_config)
transformer_kwargs = get_transformer_config_kwargs(od_config.tf_model_config, QwenImageTransformer2DModel)
self.transformer = QwenImageTransformer2DModel(od_config=od_config, **transformer_kwargs)
self.tokenizer = Qwen2Tokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
@@ -38,6 +38,7 @@ from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
QwenImageTransformer2DModel,
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs
from vllm_omni.model_executor.model_loader.weight_utils import (
download_weights_from_hf_specific,
)
@@ -231,7 +232,8 @@ class QwenImageEditPipeline(
self.vae = AutoencoderKLQwenImage.from_pretrained(model, subfolder="vae", local_files_only=local_files_only).to(
self.device
)
self.transformer = QwenImageTransformer2DModel(od_config=od_config)
transformer_kwargs = get_transformer_config_kwargs(od_config.tf_model_config, QwenImageTransformer2DModel)
self.transformer = QwenImageTransformer2DModel(od_config=od_config, **transformer_kwargs)
self.tokenizer = Qwen2Tokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
self.processor = Qwen2VLProcessor.from_pretrained(
model, subfolder="processor", local_files_only=local_files_only
@@ -41,6 +41,7 @@ from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
QwenImageTransformer2DModel,
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs
from vllm_omni.model_executor.model_loader.weight_utils import (
download_weights_from_hf_specific,
)
@@ -191,7 +192,9 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput):
self.vae = AutoencoderKLQwenImage.from_pretrained(model, subfolder="vae", local_files_only=local_files_only).to(
self.device
)
self.transformer = QwenImageTransformer2DModel(od_config=od_config)
transformer_kwargs = get_transformer_config_kwargs(od_config.tf_model_config, QwenImageTransformer2DModel)
self.transformer = QwenImageTransformer2DModel(od_config=od_config, **transformer_kwargs)
self.tokenizer = Qwen2Tokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
self.processor = Qwen2VLProcessor.from_pretrained(
model, subfolder="processor", local_files_only=local_files_only
@@ -37,6 +37,7 @@ from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
QwenImageTransformer2DModel,
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs
from vllm_omni.model_executor.model_loader.weight_utils import (
download_weights_from_hf_specific,
)
@@ -211,18 +212,8 @@ class QwenImageLayeredPipeline(nn.Module, SupportImageInput):
)
]
use_additional_t_cond = od_config.tf_model_config.use_additional_t_cond
zero_cond_t = od_config.tf_model_config.zero_cond_t
use_layer3d_rope = od_config.tf_model_config.use_layer3d_rope
guidance_embeds = od_config.tf_model_config.guidance_embeds
self.transformer = QwenImageTransformer2DModel(
od_config=od_config,
use_additional_t_cond=use_additional_t_cond,
zero_cond_t=zero_cond_t,
use_layer3d_rope=use_layer3d_rope,
guidance_embeds=guidance_embeds,
)
transformer_kwargs = get_transformer_config_kwargs(od_config.tf_model_config, QwenImageTransformer2DModel)
self.transformer = QwenImageTransformer2DModel(od_config=od_config, **transformer_kwargs)
# Pipeline configuration & processing parameters
self.vae_scale_factor = 2 ** len(self.vae.temperal_downsample) if getattr(self, "vae", None) else 8
@@ -604,7 +604,7 @@ class QwenImageTransformerBlock(nn.Module):
txt_mod1, txt_mod2 = txt_mod_params.chunk(2, dim=-1) # Each [B, 3*dim]
# Process image stream - norm1 + modulation
img_modulated, img_gate1 = self.img_norm1(hidden_states, img_mod1)
img_modulated, img_gate1 = self.img_norm1(hidden_states, img_mod1, modulate_index)
# Process text stream - norm1 + modulation
txt_modulated, txt_gate1 = self.txt_norm1(encoder_hidden_states, txt_mod1)
@@ -632,7 +632,8 @@ class QwenImageTransformerBlock(nn.Module):
encoder_hidden_states = encoder_hidden_states + txt_gate1 * txt_attn_output
# Process image stream - norm2 + MLP
img_modulated2, img_gate2 = self.img_norm2(hidden_states, img_mod2)
img_modulated2, img_gate2 = self.img_norm2(hidden_states, img_mod2, modulate_index)
img_mlp_output = self.img_mlp(img_modulated2)
hidden_states = hidden_states + img_gate2 * img_mlp_output
@@ -692,15 +693,13 @@ class QwenImageTransformer2DModel(CachedTransformer):
attention_head_dim: int = 128,
num_attention_heads: int = 24,
joint_attention_dim: int = 3584,
guidance_embeds: bool = False, # TODO: this should probably be removed
guidance_embeds: bool = False,
axes_dims_rope: tuple[int, int, int] = (16, 56, 56),
zero_cond_t: bool = False,
use_additional_t_cond: bool = False,
use_layer3d_rope: bool = False,
):
super().__init__()
model_config = od_config.tf_model_config
num_layers = model_config.num_layers
self.parallel_config = od_config.parallel_config
self.in_channels = in_channels
self.out_channels = out_channels or in_channels