From 14e5baceaf240e78d1a0c5dcc883563db23eb703 Mon Sep 17 00:00:00 2001 From: Lu Date: Tue, 19 May 2026 19:19:58 +0800 Subject: [PATCH] [Qwen-Image] Drop unused vision tower from text encoder (#3608) Signed-off-by: lulugoodcoder Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com> --- .../models/qwen_image/pipeline_qwen_image.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py index 8f05e6d0f..6f7a6bb58 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py @@ -290,7 +290,22 @@ class QwenImagePipeline(nn.Module, QwenImageCFGParallelMixin, DiffusionPipelineP ) self.text_encoder = Qwen2_5_VLForConditionalGeneration.from_pretrained( model, subfolder="text_encoder", local_files_only=local_files_only - ).to(self.device) + ) + # Qwen2.5-VL ships a vision tower that text-to-image does not use. + # Drop it while the model is still on CPU, before moving to GPU, so + # the vision tower never consumes GPU memory. Handle both transformers + # layouts: newer puts visual under .model, older puts it directly on + # the model. + visual_owner = None + if hasattr(self.text_encoder, "model") and hasattr(self.text_encoder.model, "visual"): + visual_owner = self.text_encoder.model + elif hasattr(self.text_encoder, "visual"): + visual_owner = self.text_encoder + if visual_owner is not None: + del visual_owner.visual + else: + logger.warning("Qwen-Image: vision tower not found on text encoder; skipping drop") + self.text_encoder = self.text_encoder.to(self.device) self.vae = DistributedAutoencoderKLQwenImage.from_pretrained( model, subfolder="vae", local_files_only=local_files_only ).to(self.device)