[Perf] GLM Image (#920)

Signed-off-by: JaredforReal <w13431838023@gmail.com>
Signed-off-by: Jared Wen <w13431838023@gmail.com>
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-02-27 10:29:14 +08:00
1 parent f6ed1bc4e7
commit 4146095623
25 files changed
+5182 -221

No files matched your search

@@ -17,7 +17,6 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.cache.base import CachedTransformer
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
logger = init_logger(__name__)
@@ -354,8 +353,7 @@ class GlmImageAttention(nn.Module):
nn.Dropout(0.0),
)
# RoPE and attention
self.rope = RotaryEmbedding(is_neox_style=False)
# Attention
self.attn = Attention(
num_heads=num_heads,
head_size=head_dim,
@@ -379,7 +377,6 @@ class GlmImageAttention(nn.Module):
hidden_states: Image hidden states [B, img_seq_len, D]
encoder_hidden_states: Text hidden states [B, text_seq_len, D]
image_rotary_emb: Tuple of (cos, sin) for RoPE
attention_mask: Optional attention mask for text tokens
kv_cache: Optional layer KV cache for image editing
kv_cache_mode: Cache mode (WRITE, READ, SKIP)
@@ -407,16 +404,12 @@ class GlmImageAttention(nn.Module):
# Apply RoPE only to image tokens (not text tokens)
if image_rotary_emb is not None:
cos, sin = image_rotary_emb
cos = cos.to(query.dtype)
sin = sin.to(query.dtype)
# Only apply RoPE to image part (after text_seq_length)
query_img = query[:, text_seq_length:, :, :]
key_img = key[:, text_seq_length:, :, :]
from diffusers.models.embeddings import apply_rotary_emb
query_img = apply_rotary_emb(query_img, image_rotary_emb, sequence_dim=1, use_real_unbind_dim=-2)
# key_img = self.rope(key_img, cos, sin)
key_img = apply_rotary_emb(key_img, image_rotary_emb, sequence_dim=1, use_real_unbind_dim=-2)
query = torch.cat([query[:, :text_seq_length, :, :], query_img], dim=1)
key = torch.cat([key[:, :text_seq_length, :, :], key_img], dim=1)
@@ -4,7 +4,7 @@
GlmImagePipeline implementation for vLLM-Omni.
This pipeline implements GLM-Image text-to-image generation with:
- AR stage: GlmImageForConditionalGeneration generates prior tokens
- AR stage (vLLM): GLM-Image AR stage generates prior tokens
- DiT stage: GlmImageTransformer2DModel performs diffusion denoising
- VAE: AutoencoderKL decodes latents to images
"""
@@ -31,8 +31,6 @@ from diffusers.utils.torch_utils import randn_tensor
from torch import nn
from transformers import (
ByT5Tokenizer,
GlmImageForConditionalGeneration,
GlmImageProcessor,
T5EncoderModel,
)
@@ -244,13 +242,13 @@ class GlmImagePipeline(nn.Module):
GLM-Image Pipeline for text-to-image and image-to-image generation.
This pipeline integrates:
- AR model (GlmImageForConditionalGeneration): Generates prior image tokens
- AR stage (vLLM): Generates prior image tokens
- Text encoder (T5EncoderModel): Encodes glyph/text embeddings
- DiT model (GlmImageTransformer2DModel): Diffusion transformer
- VAE (AutoencoderKL): Encodes/decodes images to/from latent space
The pipeline flow:
1. AR generates prior_token_ids from text prompt
1. AR stage provides prior_token_ids (and optionally prior_token_image_ids)
2. T5 encodes glyph text for text rendering
3. DiT performs iterative denoising conditioned on prior tokens
4. VAE decodes final latents to image
@@ -280,19 +278,6 @@ class GlmImagePipeline(nn.Module):
model_path, subfolder="scheduler", local_files_only=True
)
# Load AR model (vision_language_encoder)
logger.info("Loading GlmImageForConditionalGeneration (AR model)...")
self.vision_language_encoder = GlmImageForConditionalGeneration.from_pretrained(
model_path,
subfolder="vision_language_encoder",
local_files_only=True,
torch_dtype=torch.bfloat16,
).to(self.device)
self.vision_language_encoder.eval()
# Load processor for AR model
self.processor = GlmImageProcessor.from_pretrained(model_path, subfolder="processor", local_files_only=True)
# Load text encoder (T5 for glyph embeddings)
logger.info("Loading T5EncoderModel (glyph encoder)...")
self.text_encoder = T5EncoderModel.from_pretrained(
@@ -371,159 +356,6 @@ class GlmImagePipeline(nn.Module):
if prompt is not None and not isinstance(prompt, (str, list)):
raise ValueError(f"`prompt` must be of type `str` or `list` but is {type(prompt)}")
# ==================== AR Stage Methods ====================
@staticmethod
def _compute_generation_params(
image_grid_thw: torch.Tensor,
is_text_to_image: bool,
) -> tuple[int, int, int, int]:
"""
Compute AR generation parameters from image grid.
Args:
image_grid_thw: Image grid tensor of shape [N, 3] where each row is [t, h, w]
is_text_to_image: Whether this is text-to-image (vs image-to-image)
Returns:
Tuple of (max_new_tokens, large_image_start_offset, target_grid_h, target_grid_w)
"""
grid_sizes = []
grid_hw = []
for i in range(image_grid_thw.shape[0]):
t, h, w = image_grid_thw[i].tolist()
grid_sizes.append(int(h * w))
grid_hw.append((int(h), int(w)))
if not is_text_to_image:
# Image-to-image: only generate target image tokens
max_new_tokens = grid_sizes[-1] + 1
large_image_start_offset = 0
target_grid_h, target_grid_w = grid_hw[-1]
else:
# Text-to-image: generate both small preview and large target
total_tokens = sum(grid_sizes)
max_new_tokens = total_tokens + 1
large_image_start_offset = sum(grid_sizes[1:])
target_grid_h, target_grid_w = grid_hw[0]
return max_new_tokens, large_image_start_offset, target_grid_h, target_grid_w
@staticmethod
def _extract_large_image_tokens(
outputs: torch.Tensor, input_length: int, large_image_start_offset: int, large_image_tokens: int
) -> torch.Tensor:
"""Extract large image tokens from AR output."""
generated_tokens = outputs[0][input_length:]
large_image_start = large_image_start_offset
large_image_end = large_image_start + large_image_tokens
return generated_tokens[large_image_start:large_image_end]
@staticmethod
def _upsample_token_ids(token_ids: torch.Tensor, token_h: int, token_w: int) -> torch.Tensor:
"""Upsample token IDs by 2x using nearest neighbor interpolation."""
token_ids = token_ids.view(1, 1, token_h, token_w)
token_ids = torch.nn.functional.interpolate(token_ids.float(), scale_factor=2, mode="nearest").to(
dtype=torch.long
)
token_ids = token_ids.view(1, -1)
return token_ids
@torch.inference_mode()
def generate_prior_tokens(
self,
prompt: str,
height: int,
width: int,
image: list[PIL.Image.Image] | None = None,
factor: int = 32,
) -> tuple[torch.Tensor, list[torch.Tensor] | None]:
"""
Generate prior tokens using the AR model.
Args:
prompt: Text prompt for generation
height: Target image height
width: Target image width
image: Optional condition images for image-to-image
factor: Token factor (default 32)
Returns:
Tuple of (prior_token_ids, prior_token_image_ids)
prior_token_image_ids is a list of tensors, one per condition image
"""
device = self.vision_language_encoder.device
height = (height // factor) * factor
width = (width // factor) * factor
is_text_to_image = image is None or len(image) == 0
# Build message content
content = []
if image is not None:
for img in image:
content.append({"type": "image", "image": img})
content.append({"type": "text", "text": prompt})
messages = [{"role": "user", "content": content}]
# Apply chat template - processor will handle target dimensions and build grid
inputs = self.processor.apply_chat_template(
messages,
tokenize=True,
target_h=height,
target_w=width,
return_dict=True,
return_tensors="pt",
).to(device)
image_grid_thw = inputs.get("image_grid_thw")
# Compute generation parameters from the full grid
max_new_tokens, large_image_offset, token_h, token_w = self._compute_generation_params(
image_grid_thw=image_grid_thw, is_text_to_image=is_text_to_image
)
# Process condition images if provided
# Use image_grid_thw[:-1] to exclude the target image grid (last entry)
prior_token_image_ids = None
if image is not None and image_grid_thw is not None and len(image_grid_thw) > 1:
# Get features only for condition images (exclude target image grid)
condition_grid = image_grid_thw[:-1]
prior_token_image_embed = self.vision_language_encoder.get_image_features(
inputs["pixel_values"], condition_grid
).pooler_output
prior_token_image_embed = torch.cat(prior_token_image_embed, dim=0)
flat_prior_token_image_ids = self.vision_language_encoder.get_image_tokens(
prior_token_image_embed, condition_grid
)
# Split by image grid sizes and convert to list
split_sizes = (condition_grid.prod(dim=-1)).tolist()
prior_token_image_ids_list = torch.split(flat_prior_token_image_ids, split_sizes, dim=0)
# Convert to list with upsampling
prior_token_image_ids = []
for i, token_ids in enumerate(prior_token_image_ids_list):
grid_t, grid_h, grid_w = condition_grid[i].tolist()
token_ids = token_ids.view(1, -1)
# Upsample 2x (from d32 to d64)
token_ids_upsampled = self._upsample_token_ids(token_ids, grid_h, grid_w)
prior_token_image_ids.append(token_ids_upsampled)
# Generate with AR model
outputs = self.vision_language_encoder.generate(
**inputs,
max_new_tokens=max_new_tokens,
do_sample=True,
)
# Extract and upsample tokens
large_image_tokens = token_h * token_w
prior_token_ids_d32 = self._extract_large_image_tokens(
outputs, inputs["input_ids"].shape[-1], large_image_offset, large_image_tokens
)
prior_token_ids = self._upsample_token_ids(prior_token_ids_d32, token_h, token_w)
return prior_token_ids, prior_token_image_ids
# ==================== Text Encoding Methods ====================
def get_glyph_texts(self, prompt: str | list[str]) -> list[str]:
@@ -803,11 +635,14 @@ class GlmImagePipeline(nn.Module):
# Process each condition image through transformer to populate KV cache
for condition_image, condition_prior_token_id in zip(condition_images, prior_token_image_ids):
condition_image = condition_image.to(device=self.device, dtype=prompt_embeds.dtype)
# Move prior token ids to device (may come from CPU due to serialization)
if isinstance(condition_prior_token_id, torch.Tensor):
condition_prior_token_id = condition_prior_token_id.to(device=self.device)
# Encode condition image to latent space
# Use argmax (mode) for deterministic encoding of condition images
condition_latent = retrieve_latents(
self.vae.encode(condition_image), generator=generator, sample_mode="argmax"
self.vae.encode(condition_image.unsqueeze(0)), generator=generator, sample_mode="argmax"
)
condition_latent = (condition_latent - latents_mean) / latents_std
@@ -846,6 +681,17 @@ class GlmImagePipeline(nn.Module):
first_prompt = req.prompts[0]
prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "")
# NOTE: DiffusionEngine does an internal warmup "dummy run" during
# initialization. That request has no request_id and does not carry
# Stage-0 (AR) outputs via req.extra. For GLM-Image, we allow that
# specific warmup request to proceed by synthesizing minimal prior
# tokens, while still raising a clear error for real requests.
is_dummy_warmup = (
not getattr(req, "request_id", None)
and prompt == "dummy run"
and (req.sampling_params.num_inference_steps == 1)
)
# Get pre-computed prompt embeddings if provided
if isinstance(first_prompt, str):
prompt_embeds = None
@@ -856,20 +702,18 @@ class GlmImagePipeline(nn.Module):
# Get condition images for Image Edit mode
# Use pre-processed images from pre_process_func
preprocessed_images = (
None
if isinstance(first_prompt, str)
else [first_prompt.get("additional_information", {}).get("preprocessed_image")]
)
condition_images = (
None
if isinstance(first_prompt, str)
else first_prompt.get("additional_information", {}).get("prompt_image")
)
preprocessed_images = None
if not isinstance(first_prompt, str):
img = first_prompt.get("additional_information", {}).get("preprocessed_image")
if img is not None:
preprocessed_images = [img]
img_height = req.sampling_params.height
img_width = req.sampling_params.width
is_image_edit = preprocessed_images is not None
# Warmup may include a dummy image for image-input-capable models.
# Treat that as t2i warmup to avoid requiring i2i-only KV-cache inputs.
is_image_edit = (preprocessed_images is not None) and (not is_dummy_warmup)
# Use image dimensions as default if available
height = req.sampling_params.height or img_height or self.default_sample_size * self.vae_scale_factor
@@ -877,46 +721,67 @@ class GlmImagePipeline(nn.Module):
num_inference_steps = req.sampling_params.num_inference_steps or 50
guidance_scale = req.sampling_params.guidance_scale or 1.5
# 0. Validate inputs
self.check_inputs(prompt=prompt, height=height, width=width, prompt_embeds=prompt_embeds)
batch_size = 1
do_classifier_free_guidance = guidance_scale > 1.0
# Set seed if provided
generator = None
if req.sampling_params.seed is not None:
generator = torch.Generator(device=self.device).manual_seed(req.sampling_params.seed)
# 1. Get prior tokens - either from external source (multistage) or generate internally
# Check if prior_token_ids are provided externally (from AR stage in multistage mode)
# First try sampling_params.extra_args (for direct API usage)
# Then try prompt["extra"] (for multistage ar2diffusion output)
external_prior_tokens = req.sampling_params.extra_args.get("prior_token_ids")
external_prior_image_ids = req.sampling_params.extra_args.get("prior_token_image_ids")
if external_prior_tokens is None and isinstance(first_prompt, dict):
prompt_extra = first_prompt.get("extra", {})
external_prior_tokens = prompt_extra.get("prior_token_ids")
external_prior_image_ids = prompt_extra.get("prior_token_image_ids")
if external_prior_tokens is not None:
# Multistage mode: use externally provided prior tokens from vLLM AR stage
logger.info("Using externally provided prior tokens from AR stage...")
prior_token_id = external_prior_tokens
if isinstance(prior_token_id, list):
prior_token_id = torch.tensor(prior_token_id, dtype=torch.long, device=self.device)
elif isinstance(prior_token_id, torch.Tensor):
prior_token_id = prior_token_id.to(device=self.device, dtype=torch.long)
# Ensure shape is [1, num_tokens] for batch processing
if prior_token_id.dim() == 1:
prior_token_id = prior_token_id.unsqueeze(0)
prior_token_image_ids = external_prior_image_ids
prior_token_image_ids = None
if external_prior_image_ids is not None:
if isinstance(external_prior_image_ids, torch.Tensor):
prior_token_image_ids = [external_prior_image_ids]
else:
prior_token_image_ids = list(external_prior_image_ids)
normalized: list[torch.Tensor] = []
for item in prior_token_image_ids:
if isinstance(item, torch.Tensor):
tensor_item = item.to(device=self.device, dtype=torch.long)
else:
tensor_item = torch.tensor(item, dtype=torch.long, device=self.device)
if tensor_item.dim() == 1:
tensor_item = tensor_item.unsqueeze(0)
normalized.append(tensor_item)
prior_token_image_ids = normalized
elif is_dummy_warmup:
# Synthesize a minimal, shape-correct prior token sequence.
# The diffusion transformer expects prior_token_id shape:
# [B, (H_lat/p)*(W_lat/p)] where H_lat=H/vae_scale_factor.
h_lat = height // self.vae_scale_factor
w_lat = width // self.vae_scale_factor
seq_len = (h_lat * w_lat) // (self._patch_size**2)
prior_token_id = torch.zeros((1, seq_len), dtype=torch.long, device=self.device)
prior_token_image_ids = None
else:
# Single-stage mode: generate prior tokens with internal AR model
logger.info("Generating prior tokens with AR model...")
prior_token_id, prior_token_image_ids = self.generate_prior_tokens(
prompt=prompt,
image=condition_images,
height=height,
width=width,
raise ValueError(
"GLM-Image diffusion pipeline expects prior tokens from the vLLM AR stage. "
"Pass them via req.extra['prior_token_ids'] (and for i2i, req.extra['prior_token_image_ids'])."
)
# 2. Encode prompt for glyph embeddings
logger.info("Encoding prompt...")
prompt_embeds, negative_prompt_embeds = self.encode_prompt(
prompt,
do_classifier_free_guidance=do_classifier_free_guidance,
@@ -928,15 +793,17 @@ class GlmImagePipeline(nn.Module):
# 3. Prepare KV cache for Image Edit mode
kv_caches = None
if is_image_edit and prior_token_image_ids is None:
raise ValueError(
"Image edit (i2i) requires req.extra['prior_token_image_ids'] from the AR stage to build KV cache."
)
if is_image_edit and prior_token_image_ids is not None:
logger.info("Preparing KV cache for Image Edit mode...")
kv_caches = self._prepare_condition_image_kv_cache(
condition_images=preprocessed_images,
prior_token_image_ids=prior_token_image_ids,
prompt_embeds=prompt_embeds,
generator=generator,
)
# Switch to read mode for denoising
kv_caches.set_mode("read")
# 4. Prepare latents
@@ -971,8 +838,7 @@ class GlmImagePipeline(nn.Module):
target_size = torch.tensor([[height, width]], dtype=prompt_embeds.dtype, device=self.device)
crop_coords = torch.zeros((1, 2), dtype=prompt_embeds.dtype, device=self.device)
# 7. Denoising loop with CFG-parallel support
logger.info(f"Starting denoising loop with {num_inference_steps} steps...")
# 7. Denoising loop
latents = self.diffuse(
latents=latents,
prior_token_id=prior_token_id,
@@ -987,7 +853,6 @@ class GlmImagePipeline(nn.Module):
)
# 8. VAE decode
logger.info("Decoding latents with VAE...")
latents = latents.to(self.vae.dtype)
latents_mean = (
torch.tensor(self.vae.config.latents_mean)
@@ -1002,8 +867,6 @@ class GlmImagePipeline(nn.Module):
latents = latents * latents_std + latents_mean
image = self.vae.decode(latents, return_dict=False, generator=generator)[0]
# 9. Leave post-process to vllm-omni pipeline
return DiffusionOutput(output=image)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: