[Diffusion] Non-Intrusive Sequence Parallelism (SP) Model Support Abstraction for vLLM-Omni Framework (#779)

Signed-off-by: mxuax <mxuax@connect.ust.hk>
Signed-off-by: XU Mingshi <91017482+mxuax@users.noreply.github.com>
This commit is contained in:
XU Mingshi authored and GitHub committed 2026-01-21 14:43:09 +08:00
1 parent 334d306d9a
commit 25b08b3371
17 files changed
+3369 -479

No files matched your search

@@ -27,14 +27,11 @@ from vllm_omni.diffusion.attention.backends.abstract import (
AttentionMetadata,
)
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.attention.selector import _BACKENDS_SUPPORT_ATTENTION_MASK, get_attn_backend
from vllm_omni.diffusion.cache.base import CachedTransformer
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.distributed.parallel_state import (
get_ring_parallel_world_size,
get_sequence_parallel_rank,
get_sequence_parallel_world_size,
get_sp_group,
from vllm_omni.diffusion.distributed.sp_plan import (
SequenceParallelInput,
SequenceParallelOutput,
)
from vllm_omni.diffusion.forward_context import get_forward_context
from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNorm
@@ -43,6 +40,55 @@ from vllm_omni.diffusion.layers.rope import RotaryEmbedding
logger = init_logger(__name__)
class ImageRopePrepare(nn.Module):
"""Prepares image hidden_states and RoPE embeddings for sequence parallel.
This module encapsulates the input linear projection and RoPE computation.
Similar to Z-Image's UnifiedPrepare, this creates a module boundary where
_sp_plan can shard outputs via split_output=True.
The key insight is that hidden_states and vid_freqs must be sharded together
to maintain dimension alignment for RoPE computation in attention layers.
Note: Our _sp_plan corresponds to diffusers' _cp_plan (Context Parallelism).
"""
def __init__(self, img_in: nn.Linear, pos_embed: nn.Module):
super().__init__()
self.img_in = img_in
self.pos_embed = pos_embed
def forward(
self,
hidden_states: torch.Tensor,
img_shapes: list[tuple[int, int, int]],
txt_seq_lens: list[int],
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Prepare hidden_states and RoPE for SP.
Args:
hidden_states: [batch, img_seq_len, channels]
img_shapes: List of (frame, height, width) tuples
txt_seq_lens: List of text sequence lengths
Returns:
hidden_states: Processed hidden states [batch, img_seq_len, dim]
vid_freqs: Image RoPE frequencies [img_seq_len, rope_dim]
txt_freqs: Text RoPE frequencies [txt_seq_len, rope_dim]
Note: _sp_plan will shard hidden_states and vid_freqs via split_output=True
txt_freqs is kept replicated for dual-stream attention
"""
# Apply input projection
hidden_states = self.img_in(hidden_states)
# Compute RoPE embeddings
image_rotary_emb = self.pos_embed(img_shapes, txt_seq_lens, device=hidden_states.device)
vid_freqs, txt_freqs = image_rotary_emb
return hidden_states, vid_freqs, txt_freqs
class QwenTimestepProjEmbeddings(nn.Module):
def __init__(self, embedding_dim, use_additional_t_cond=False):
super().__init__()
@@ -436,6 +482,25 @@ class QwenImageCrossAttention(nn.Module):
hidden_states_mask: torch.Tensor | None = None,
encoder_hidden_states_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Check for SP auto_pad: create attention mask dynamically if padding was applied
# In Ulysses mode, attention is computed on the FULL sequence (after All-to-All)
if (
self.parallel_config is not None
and self.parallel_config.sequence_parallel_size > 1
and hidden_states_mask is None
):
ctx = get_forward_context()
if ctx.sp_original_seq_len is not None and ctx.sp_padding_size > 0:
# Create mask for the full (padded) sequence
# valid positions = True, padding positions = False
batch_size = hidden_states.shape[0]
padded_seq_len = ctx.sp_original_seq_len + ctx.sp_padding_size
hidden_states_mask = torch.ones(
batch_size, padded_seq_len, dtype=torch.bool, device=hidden_states.device
)
hidden_states_mask[:, ctx.sp_original_seq_len :] = False
# if mask is all true, set it to None
if hidden_states_mask is not None and hidden_states_mask.all():
hidden_states_mask = None
if encoder_hidden_states_mask is not None and encoder_hidden_states_mask.all():
@@ -718,6 +783,31 @@ class QwenImageTransformer2DModel(CachedTransformer):
# used for torch compile optimizations
_repeated_blocks = ["QwenImageTransformerBlock"]
# Sequence Parallelism plan (following diffusers' _cp_plan pattern)
# Similar to Z-Image's UnifiedPrepare, we use ImageRopePrepare to create
# a module boundary where _sp_plan can shard hidden_states and vid_freqs together.
#
# Key insight: hidden_states and vid_freqs MUST be sharded together to maintain
# dimension alignment for RoPE computation in attention layers.
#
# auto_pad=True enables automatic padding when sequence length is not divisible
# by SP world size. This creates an attention mask stored in ForwardContext
# that attention layers can use to ignore padding positions.
#
# Note: _sp_plan corresponds to diffusers' _cp_plan (Context Parallelism)
_sp_plan = {
# Shard ImageRopePrepare outputs (hidden_states and vid_freqs must be sharded together)
"image_rope_prepare": {
# hidden_states: auto_pad=True for variable sequence length support
0: SequenceParallelInput(split_dim=1, expected_dims=3, split_output=True, auto_pad=True),
# vid_freqs: auto_pad=True to match hidden_states padding
1: SequenceParallelInput(split_dim=0, expected_dims=2, split_output=True, auto_pad=True),
# txt_freqs (index 2) is NOT sharded - kept replicated for dual-stream attention
},
# Gather output at proj_out
"proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
}
def __init__(
self,
od_config: OmniDiffusionConfig,
@@ -773,6 +863,10 @@ class QwenImageTransformer2DModel(CachedTransformer):
self.gradient_checkpointing = False
self.zero_cond_t = zero_cond_t
# ImageRopePrepare module for _sp_plan to shard hidden_states and vid_freqs together
# This ensures RoPE dimensions align with hidden_states after sharding
self.image_rope_prepare = ImageRopePrepare(self.img_in, self.pos_embed)
def forward(
self,
hidden_states: torch.Tensor,
@@ -816,49 +910,17 @@ class QwenImageTransformer2DModel(CachedTransformer):
# else:
# lora_scale = 1.0
original_seq_len = None
seq_padding = 0
hidden_states_mask = None
# Set split_text_embed_in_sp = False for dual-stream attention
# QwenImage uses *dual-stream* (text + image) and runs a *joint attention*.
# Text embeddings must be replicated across SP ranks for correctness.
if self.parallel_config.sequence_parallel_size > 1:
batch_size, seq_len, channels = hidden_states.shape
sp_size = get_sequence_parallel_world_size()
if seq_len % sp_size != 0:
# ring_attn, sage_attn do not support attention_mask
if get_attn_backend(-1).get_name() not in _BACKENDS_SUPPORT_ATTENTION_MASK:
raise ValueError(
f"When generating image shape that the sequence length is NOT divisible by sp_size={sp_size},"
f"cannot use {get_attn_backend(-1).get_name()} which does not support attention_mask."
f"Please switch to SDPA or Ascend attention backend."
)
# ring attention does not support attention_mask
if get_ring_parallel_world_size() > 1:
raise ValueError(
f"When generating image shape that the sequence length is NOT divisible by sp_size={sp_size},"
f"cannot use ring attention which does not support attention_mask."
f"Please switch to Ulysses SP only."
)
seq_padding = sp_size - (seq_len % sp_size)
original_seq_len = seq_len
hidden_states_mask = torch.ones(
batch_size, seq_len + seq_padding, dtype=torch.bool, device=hidden_states.device
)
hidden_states_mask[:, seq_len:] = False
padding_tensor = torch.zeros(
batch_size, seq_padding, channels, dtype=hidden_states.dtype, device=hidden_states.device
)
hidden_states = torch.cat([hidden_states, padding_tensor], dim=1)
hidden_states = torch.chunk(hidden_states, sp_size, dim=-2)[get_sequence_parallel_rank()]
# NOTE:
# QwenImage uses *dual-stream* (text + image) and runs a *joint attention*.
# text embeddings to be replicated across SP ranks for correctness.
get_forward_context().split_text_embed_in_sp = False
hidden_states = self.img_in(hidden_states)
# Prepare hidden_states and RoPE via ImageRopePrepare module
# _sp_plan will shard hidden_states and vid_freqs together via split_output=True
# txt_freqs is kept replicated for dual-stream attention
hidden_states, vid_freqs, txt_freqs = self.image_rope_prepare(hidden_states, img_shapes, txt_seq_lens)
image_rotary_emb = (vid_freqs, txt_freqs)
# Ensure timestep tensor is on the same device and dtype as hidden_states
timestep = timestep.to(device=hidden_states.device, dtype=hidden_states.dtype)
@@ -885,23 +947,6 @@ class QwenImageTransformer2DModel(CachedTransformer):
else self.time_text_embed(timestep, guidance, hidden_states, additional_t_cond)
)
image_rotary_emb = self.pos_embed(img_shapes, txt_seq_lens, device=hidden_states.device)
def get_rotary_emb_chunk(freqs, padding=0):
# Pad rotary embeddings if needed
if padding > 0:
padding_tensor = torch.zeros(padding, freqs.shape[-1], dtype=freqs.dtype, device=freqs.device)
freqs = torch.cat([freqs, padding_tensor], dim=0)
freqs = torch.chunk(freqs, get_sequence_parallel_world_size(), dim=0)[get_sequence_parallel_rank()]
return freqs
if self.parallel_config.sequence_parallel_size > 1:
img_freqs, txt_freqs = image_rotary_emb
img_freqs = get_rotary_emb_chunk(img_freqs, seq_padding)
if get_forward_context().split_text_embed_in_sp:
txt_freqs = get_rotary_emb_chunk(txt_freqs)
image_rotary_emb = (img_freqs, txt_freqs)
for index_block, block in enumerate(self.transformer_blocks):
encoder_hidden_states, hidden_states = block(
hidden_states=hidden_states,
@@ -911,7 +956,6 @@ class QwenImageTransformer2DModel(CachedTransformer):
image_rotary_emb=image_rotary_emb,
joint_attention_kwargs=attention_kwargs,
modulate_index=modulate_index,
hidden_states_mask=hidden_states_mask,
)
if self.zero_cond_t:
@@ -920,14 +964,8 @@ class QwenImageTransformer2DModel(CachedTransformer):
hidden_states = self.norm_out(hidden_states, temb)
output = self.proj_out(hidden_states)
if self.parallel_config.sequence_parallel_size > 1:
output = get_sp_group().all_gather(output, dim=-2)
# Remove padding if it was added
if original_seq_len is not None:
output = output[:, :original_seq_len, :]
torch.cuda.empty_cache()
# Note: SP gather is handled automatically by _sp_plan's SequenceParallelGatherHook
# on proj_out output. No manual all_gather needed here.
return Transformer2DModelOutput(sample=output)
@@ -176,6 +176,9 @@ class ZImagePipeline(nn.Module):
self.transformer = ZImageTransformer2DModel()
self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
# Note: Context parallelism is applied centrally in registry.initialize_model()
# following diffusers' pattern of enable_parallelism() at model loading time
self.vae_scale_factor = (
2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8
)
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# _sp_plan definition adapted from HuggingFace diffusers library (_cp_plan)
# Copyright 2025 Alibaba Z-Image Team and The HuggingFace Team. All rights reserved.
#
@@ -32,7 +33,14 @@ from vllm.model_executor.layers.linear import (
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.forward_context import get_forward_context, is_forward_context_available
from vllm_omni.diffusion.distributed.sp_plan import (
SequenceParallelInput,
SequenceParallelOutput,
)
from vllm_omni.diffusion.forward_context import (
get_forward_context,
is_forward_context_available,
)
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
ADALN_EMBED_DIM = 256
@@ -41,6 +49,63 @@ SEQ_MULTI_OF = 32
logger = init_logger(__name__)
class UnifiedPrepare(nn.Module):
"""Prepares unified tensors for transformer blocks.
This module encapsulates the unification of x and cap tensors into unified
sequences. Similar to how Wan's `rope` module outputs rotary embeddings,
this module outputs unified tensors that can be sharded via _sp_plan's
split_output=True mechanism.
This follows the diffusers pattern where tensor preparation happens in
a dedicated submodule, enabling _sp_plan hooks to work at module boundaries.
"""
def forward(
self,
x: torch.Tensor,
x_cos: torch.Tensor,
x_sin: torch.Tensor,
cap_feats: torch.Tensor,
cap_cos: torch.Tensor,
cap_sin: torch.Tensor,
x_item_seqlens: list[int],
cap_item_seqlens: list[int],
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Combine x and cap tensors into unified sequences.
Returns:
unified: Combined hidden states [batch, seq_len, dim]
unified_cos: Combined RoPE cos [batch, seq_len, rope_dim]
unified_sin: Combined RoPE sin [batch, seq_len, rope_dim]
unified_attn_mask: Combined attention mask [batch, seq_len]
"""
bsz = x.shape[0]
device = x.device
unified = []
unified_cos = []
unified_sin = []
for i in range(bsz):
x_len = x_item_seqlens[i]
cap_len = cap_item_seqlens[i]
unified.append(torch.cat([x[i][:x_len], cap_feats[i][:cap_len]]))
unified_cos.append(torch.cat([x_cos[i][:x_len], cap_cos[i][:cap_len]]))
unified_sin.append(torch.cat([x_sin[i][:x_len], cap_sin[i][:cap_len]]))
unified_item_seqlens = [a + b for a, b in zip(cap_item_seqlens, x_item_seqlens)]
unified_max_item_seqlen = max(unified_item_seqlens)
unified = pad_sequence(unified, batch_first=True, padding_value=0.0)
unified_cos = pad_sequence(unified_cos, batch_first=True, padding_value=0.0)
unified_sin = pad_sequence(unified_sin, batch_first=True, padding_value=0.0)
unified_attn_mask = torch.zeros((bsz, unified_max_item_seqlen), dtype=torch.bool, device=device)
for i, seq_len in enumerate(unified_item_seqlens):
unified_attn_mask[i, :seq_len] = 1
return unified, unified_cos, unified_sin, unified_attn_mask
def _positive_divisors(n: int) -> set[int]:
if n <= 0:
return set()
@@ -456,8 +521,52 @@ class RopeEmbedder:
class ZImageTransformer2DModel(nn.Module):
"""Z-Image Transformer model for image generation.
Sequence Parallelism:
This model supports non-intrusive SP via _sp_plan. The plan specifies:
- Input splitting at first main transformer block (unified sequence)
- RoPE (cos/sin) splitting along sequence dimension
- Attention mask splitting along sequence dimension
- Output gathering at final_layer
The SP is applied to the main `layers` transformer blocks where the
unified image+caption sequence is processed jointly.
Note: noise_refiner and context_refiner are NOT parallelized as they
process image and caption separately before unification.
Important: The default _sp_plan assumes patch_size=2 and f_patch_size=1.
If using different patch configurations, update _sp_plan accordingly.
Note: Our "Sequence Parallelism" (SP) corresponds to "Context Parallelism" (CP) in diffusers.
"""
_repeated_blocks = ["ZImageTransformerBlock"]
# Sequence Parallelism for Z-Image (following diffusers' _cp_plan pattern)
# Similar to how Wan uses `rope` module's split_output to shard rotary embeddings,
# Z-Image uses `unified_prepare` module's split_output to shard unified tensors.
#
# The _sp_plan specifies sharding/gathering at module boundaries:
# - unified_prepare: Split all 4 outputs (unified, cos, sin, attn_mask) via split_output=True
# - layers.0: hidden_states input is already sharded from unified_prepare output
# - all_final_layer.2-1: Gather outputs after the final layer
#
# Note: _sp_plan corresponds to diffusers' _cp_plan (Context Parallelism)
_sp_plan = {
# Shard unified_prepare outputs (similar to Wan's rope module)
# This shards all 4 return values: unified, unified_cos, unified_sin, unified_attn_mask
"unified_prepare": {
0: SequenceParallelInput(split_dim=1, expected_dims=3, split_output=True), # unified
1: SequenceParallelInput(split_dim=1, expected_dims=3, split_output=True), # unified_cos
2: SequenceParallelInput(split_dim=1, expected_dims=3, split_output=True), # unified_sin
3: SequenceParallelInput(split_dim=1, expected_dims=2, split_output=True), # unified_attn_mask
},
# Gather output at final_layer (default: patch_size=2, f_patch_size=1)
"all_final_layer.2-1": SequenceParallelOutput(gather_dim=1, expected_dims=3),
}
def __init__(
self,
all_patch_size=(2,),
@@ -572,6 +681,11 @@ class ZImageTransformer2DModel(nn.Module):
self.rope_embedder = RopeEmbedder(theta=rope_theta, axes_dims=axes_dims, axes_lens=axes_lens)
# UnifiedPrepare module for combining x and cap tensors
# This enables _cp_plan to shard outputs via split_output=True
# Similar to how Wan's rope module enables rotary embedding sharding
self.unified_prepare = UnifiedPrepare()
def unpatchify(self, x: list[torch.Tensor], size: list[tuple], patch_size, f_patch_size) -> list[torch.Tensor]:
pH = pW = patch_size
pF = f_patch_size
@@ -772,31 +886,19 @@ class ZImageTransformer2DModel(nn.Module):
for layer in self.context_refiner:
cap_feats = layer(cap_feats, cap_attn_mask, cap_cos, cap_sin)
# unified
unified = []
unified_cos = []
unified_sin = []
for i in range(bsz):
x_len = x_item_seqlens[i]
cap_len = cap_item_seqlens[i]
unified.append(torch.cat([x[i][:x_len], cap_feats[i][:cap_len]]))
unified_cos.append(torch.cat([x_cos[i][:x_len], cap_cos[i][:cap_len]]))
unified_sin.append(torch.cat([x_sin[i][:x_len], cap_sin[i][:cap_len]]))
unified_item_seqlens = [a + b for a, b in zip(cap_item_seqlens, x_item_seqlens)]
assert unified_item_seqlens == [len(_) for _ in unified]
unified_max_item_seqlen = max(unified_item_seqlens)
unified = pad_sequence(unified, batch_first=True, padding_value=0.0)
unified_cos = pad_sequence(unified_cos, batch_first=True, padding_value=0.0)
unified_sin = pad_sequence(unified_sin, batch_first=True, padding_value=0.0)
unified_attn_mask = torch.zeros((bsz, unified_max_item_seqlen), dtype=torch.bool, device=device)
for i, seq_len in enumerate(unified_item_seqlens):
unified_attn_mask[i, :seq_len] = 1
# Prepare unified tensors via UnifiedPrepare module
# This enables _cp_plan to shard outputs via split_output=True
unified, unified_cos, unified_sin, unified_attn_mask = self.unified_prepare(
x, x_cos, x_sin, cap_feats, cap_cos, cap_sin, x_item_seqlens, cap_item_seqlens
)
# Main transformer blocks
for layer in self.layers:
unified = layer(unified, unified_attn_mask, unified_cos, unified_sin, adaln_input)
# Final layer
unified = self.all_final_layer[f"{patch_size}-{f_patch_size}"](unified, adaln_input)
unified = list(unified.unbind(dim=0))
x = self.unpatchify(unified, x_size, patch_size, f_patch_size)