[Diffusion] Non-Intrusive Sequence Parallelism (SP) Model Support Abstraction for vLLM-Omni Framework (#779)
Signed-off-by: mxuax <mxuax@connect.ust.hk> Signed-off-by: XU Mingshi <91017482+mxuax@users.noreply.github.com>
This commit is contained in:
1 parent
334d306d9a
commit
25b08b3371
17 files changed
+3369
-479
No files matched your search
@@ -27,14 +27,11 @@ from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionMetadata,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.attention.selector import _BACKENDS_SUPPORT_ATTENTION_MASK, get_attn_backend
|
||||
from vllm_omni.diffusion.cache.base import CachedTransformer
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
get_ring_parallel_world_size,
|
||||
get_sequence_parallel_rank,
|
||||
get_sequence_parallel_world_size,
|
||||
get_sp_group,
|
||||
from vllm_omni.diffusion.distributed.sp_plan import (
|
||||
SequenceParallelInput,
|
||||
SequenceParallelOutput,
|
||||
)
|
||||
from vllm_omni.diffusion.forward_context import get_forward_context
|
||||
from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNorm
|
||||
@@ -43,6 +40,55 @@ from vllm_omni.diffusion.layers.rope import RotaryEmbedding
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ImageRopePrepare(nn.Module):
|
||||
"""Prepares image hidden_states and RoPE embeddings for sequence parallel.
|
||||
|
||||
This module encapsulates the input linear projection and RoPE computation.
|
||||
Similar to Z-Image's UnifiedPrepare, this creates a module boundary where
|
||||
_sp_plan can shard outputs via split_output=True.
|
||||
|
||||
The key insight is that hidden_states and vid_freqs must be sharded together
|
||||
to maintain dimension alignment for RoPE computation in attention layers.
|
||||
|
||||
Note: Our _sp_plan corresponds to diffusers' _cp_plan (Context Parallelism).
|
||||
"""
|
||||
|
||||
def __init__(self, img_in: nn.Linear, pos_embed: nn.Module):
|
||||
super().__init__()
|
||||
self.img_in = img_in
|
||||
self.pos_embed = pos_embed
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
img_shapes: list[tuple[int, int, int]],
|
||||
txt_seq_lens: list[int],
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Prepare hidden_states and RoPE for SP.
|
||||
|
||||
Args:
|
||||
hidden_states: [batch, img_seq_len, channels]
|
||||
img_shapes: List of (frame, height, width) tuples
|
||||
txt_seq_lens: List of text sequence lengths
|
||||
|
||||
Returns:
|
||||
hidden_states: Processed hidden states [batch, img_seq_len, dim]
|
||||
vid_freqs: Image RoPE frequencies [img_seq_len, rope_dim]
|
||||
txt_freqs: Text RoPE frequencies [txt_seq_len, rope_dim]
|
||||
|
||||
Note: _sp_plan will shard hidden_states and vid_freqs via split_output=True
|
||||
txt_freqs is kept replicated for dual-stream attention
|
||||
"""
|
||||
# Apply input projection
|
||||
hidden_states = self.img_in(hidden_states)
|
||||
|
||||
# Compute RoPE embeddings
|
||||
image_rotary_emb = self.pos_embed(img_shapes, txt_seq_lens, device=hidden_states.device)
|
||||
vid_freqs, txt_freqs = image_rotary_emb
|
||||
|
||||
return hidden_states, vid_freqs, txt_freqs
|
||||
|
||||
|
||||
class QwenTimestepProjEmbeddings(nn.Module):
|
||||
def __init__(self, embedding_dim, use_additional_t_cond=False):
|
||||
super().__init__()
|
||||
@@ -436,6 +482,25 @@ class QwenImageCrossAttention(nn.Module):
|
||||
hidden_states_mask: torch.Tensor | None = None,
|
||||
encoder_hidden_states_mask: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Check for SP auto_pad: create attention mask dynamically if padding was applied
|
||||
# In Ulysses mode, attention is computed on the FULL sequence (after All-to-All)
|
||||
if (
|
||||
self.parallel_config is not None
|
||||
and self.parallel_config.sequence_parallel_size > 1
|
||||
and hidden_states_mask is None
|
||||
):
|
||||
ctx = get_forward_context()
|
||||
if ctx.sp_original_seq_len is not None and ctx.sp_padding_size > 0:
|
||||
# Create mask for the full (padded) sequence
|
||||
# valid positions = True, padding positions = False
|
||||
batch_size = hidden_states.shape[0]
|
||||
padded_seq_len = ctx.sp_original_seq_len + ctx.sp_padding_size
|
||||
hidden_states_mask = torch.ones(
|
||||
batch_size, padded_seq_len, dtype=torch.bool, device=hidden_states.device
|
||||
)
|
||||
hidden_states_mask[:, ctx.sp_original_seq_len :] = False
|
||||
|
||||
# if mask is all true, set it to None
|
||||
if hidden_states_mask is not None and hidden_states_mask.all():
|
||||
hidden_states_mask = None
|
||||
if encoder_hidden_states_mask is not None and encoder_hidden_states_mask.all():
|
||||
@@ -718,6 +783,31 @@ class QwenImageTransformer2DModel(CachedTransformer):
|
||||
# used for torch compile optimizations
|
||||
_repeated_blocks = ["QwenImageTransformerBlock"]
|
||||
|
||||
# Sequence Parallelism plan (following diffusers' _cp_plan pattern)
|
||||
# Similar to Z-Image's UnifiedPrepare, we use ImageRopePrepare to create
|
||||
# a module boundary where _sp_plan can shard hidden_states and vid_freqs together.
|
||||
#
|
||||
# Key insight: hidden_states and vid_freqs MUST be sharded together to maintain
|
||||
# dimension alignment for RoPE computation in attention layers.
|
||||
#
|
||||
# auto_pad=True enables automatic padding when sequence length is not divisible
|
||||
# by SP world size. This creates an attention mask stored in ForwardContext
|
||||
# that attention layers can use to ignore padding positions.
|
||||
#
|
||||
# Note: _sp_plan corresponds to diffusers' _cp_plan (Context Parallelism)
|
||||
_sp_plan = {
|
||||
# Shard ImageRopePrepare outputs (hidden_states and vid_freqs must be sharded together)
|
||||
"image_rope_prepare": {
|
||||
# hidden_states: auto_pad=True for variable sequence length support
|
||||
0: SequenceParallelInput(split_dim=1, expected_dims=3, split_output=True, auto_pad=True),
|
||||
# vid_freqs: auto_pad=True to match hidden_states padding
|
||||
1: SequenceParallelInput(split_dim=0, expected_dims=2, split_output=True, auto_pad=True),
|
||||
# txt_freqs (index 2) is NOT sharded - kept replicated for dual-stream attention
|
||||
},
|
||||
# Gather output at proj_out
|
||||
"proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
od_config: OmniDiffusionConfig,
|
||||
@@ -773,6 +863,10 @@ class QwenImageTransformer2DModel(CachedTransformer):
|
||||
self.gradient_checkpointing = False
|
||||
self.zero_cond_t = zero_cond_t
|
||||
|
||||
# ImageRopePrepare module for _sp_plan to shard hidden_states and vid_freqs together
|
||||
# This ensures RoPE dimensions align with hidden_states after sharding
|
||||
self.image_rope_prepare = ImageRopePrepare(self.img_in, self.pos_embed)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -816,49 +910,17 @@ class QwenImageTransformer2DModel(CachedTransformer):
|
||||
# else:
|
||||
# lora_scale = 1.0
|
||||
|
||||
original_seq_len = None
|
||||
seq_padding = 0
|
||||
hidden_states_mask = None
|
||||
|
||||
# Set split_text_embed_in_sp = False for dual-stream attention
|
||||
# QwenImage uses *dual-stream* (text + image) and runs a *joint attention*.
|
||||
# Text embeddings must be replicated across SP ranks for correctness.
|
||||
if self.parallel_config.sequence_parallel_size > 1:
|
||||
batch_size, seq_len, channels = hidden_states.shape
|
||||
sp_size = get_sequence_parallel_world_size()
|
||||
|
||||
if seq_len % sp_size != 0:
|
||||
# ring_attn, sage_attn do not support attention_mask
|
||||
if get_attn_backend(-1).get_name() not in _BACKENDS_SUPPORT_ATTENTION_MASK:
|
||||
raise ValueError(
|
||||
f"When generating image shape that the sequence length is NOT divisible by sp_size={sp_size},"
|
||||
f"cannot use {get_attn_backend(-1).get_name()} which does not support attention_mask."
|
||||
f"Please switch to SDPA or Ascend attention backend."
|
||||
)
|
||||
# ring attention does not support attention_mask
|
||||
if get_ring_parallel_world_size() > 1:
|
||||
raise ValueError(
|
||||
f"When generating image shape that the sequence length is NOT divisible by sp_size={sp_size},"
|
||||
f"cannot use ring attention which does not support attention_mask."
|
||||
f"Please switch to Ulysses SP only."
|
||||
)
|
||||
|
||||
seq_padding = sp_size - (seq_len % sp_size)
|
||||
original_seq_len = seq_len
|
||||
|
||||
hidden_states_mask = torch.ones(
|
||||
batch_size, seq_len + seq_padding, dtype=torch.bool, device=hidden_states.device
|
||||
)
|
||||
hidden_states_mask[:, seq_len:] = False
|
||||
padding_tensor = torch.zeros(
|
||||
batch_size, seq_padding, channels, dtype=hidden_states.dtype, device=hidden_states.device
|
||||
)
|
||||
hidden_states = torch.cat([hidden_states, padding_tensor], dim=1)
|
||||
|
||||
hidden_states = torch.chunk(hidden_states, sp_size, dim=-2)[get_sequence_parallel_rank()]
|
||||
# NOTE:
|
||||
# QwenImage uses *dual-stream* (text + image) and runs a *joint attention*.
|
||||
# text embeddings to be replicated across SP ranks for correctness.
|
||||
get_forward_context().split_text_embed_in_sp = False
|
||||
|
||||
hidden_states = self.img_in(hidden_states)
|
||||
# Prepare hidden_states and RoPE via ImageRopePrepare module
|
||||
# _sp_plan will shard hidden_states and vid_freqs together via split_output=True
|
||||
# txt_freqs is kept replicated for dual-stream attention
|
||||
hidden_states, vid_freqs, txt_freqs = self.image_rope_prepare(hidden_states, img_shapes, txt_seq_lens)
|
||||
image_rotary_emb = (vid_freqs, txt_freqs)
|
||||
|
||||
# Ensure timestep tensor is on the same device and dtype as hidden_states
|
||||
timestep = timestep.to(device=hidden_states.device, dtype=hidden_states.dtype)
|
||||
@@ -885,23 +947,6 @@ class QwenImageTransformer2DModel(CachedTransformer):
|
||||
else self.time_text_embed(timestep, guidance, hidden_states, additional_t_cond)
|
||||
)
|
||||
|
||||
image_rotary_emb = self.pos_embed(img_shapes, txt_seq_lens, device=hidden_states.device)
|
||||
|
||||
def get_rotary_emb_chunk(freqs, padding=0):
|
||||
# Pad rotary embeddings if needed
|
||||
if padding > 0:
|
||||
padding_tensor = torch.zeros(padding, freqs.shape[-1], dtype=freqs.dtype, device=freqs.device)
|
||||
freqs = torch.cat([freqs, padding_tensor], dim=0)
|
||||
freqs = torch.chunk(freqs, get_sequence_parallel_world_size(), dim=0)[get_sequence_parallel_rank()]
|
||||
return freqs
|
||||
|
||||
if self.parallel_config.sequence_parallel_size > 1:
|
||||
img_freqs, txt_freqs = image_rotary_emb
|
||||
img_freqs = get_rotary_emb_chunk(img_freqs, seq_padding)
|
||||
if get_forward_context().split_text_embed_in_sp:
|
||||
txt_freqs = get_rotary_emb_chunk(txt_freqs)
|
||||
image_rotary_emb = (img_freqs, txt_freqs)
|
||||
|
||||
for index_block, block in enumerate(self.transformer_blocks):
|
||||
encoder_hidden_states, hidden_states = block(
|
||||
hidden_states=hidden_states,
|
||||
@@ -911,7 +956,6 @@ class QwenImageTransformer2DModel(CachedTransformer):
|
||||
image_rotary_emb=image_rotary_emb,
|
||||
joint_attention_kwargs=attention_kwargs,
|
||||
modulate_index=modulate_index,
|
||||
hidden_states_mask=hidden_states_mask,
|
||||
)
|
||||
|
||||
if self.zero_cond_t:
|
||||
@@ -920,14 +964,8 @@ class QwenImageTransformer2DModel(CachedTransformer):
|
||||
hidden_states = self.norm_out(hidden_states, temb)
|
||||
output = self.proj_out(hidden_states)
|
||||
|
||||
if self.parallel_config.sequence_parallel_size > 1:
|
||||
output = get_sp_group().all_gather(output, dim=-2)
|
||||
|
||||
# Remove padding if it was added
|
||||
if original_seq_len is not None:
|
||||
output = output[:, :original_seq_len, :]
|
||||
|
||||
torch.cuda.empty_cache()
|
||||
# Note: SP gather is handled automatically by _sp_plan's SequenceParallelGatherHook
|
||||
# on proj_out output. No manual all_gather needed here.
|
||||
|
||||
return Transformer2DModelOutput(sample=output)
|
||||
|
||||
|
||||
@@ -176,6 +176,9 @@ class ZImagePipeline(nn.Module):
|
||||
self.transformer = ZImageTransformer2DModel()
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
|
||||
|
||||
# Note: Context parallelism is applied centrally in registry.initialize_model()
|
||||
# following diffusers' pattern of enable_parallelism() at model loading time
|
||||
|
||||
self.vae_scale_factor = (
|
||||
2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8
|
||||
)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# _sp_plan definition adapted from HuggingFace diffusers library (_cp_plan)
|
||||
|
||||
# Copyright 2025 Alibaba Z-Image Team and The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
@@ -32,7 +33,14 @@ from vllm.model_executor.layers.linear import (
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm_omni.diffusion.distributed.sp_plan import (
|
||||
SequenceParallelInput,
|
||||
SequenceParallelOutput,
|
||||
)
|
||||
from vllm_omni.diffusion.forward_context import (
|
||||
get_forward_context,
|
||||
is_forward_context_available,
|
||||
)
|
||||
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
|
||||
|
||||
ADALN_EMBED_DIM = 256
|
||||
@@ -41,6 +49,63 @@ SEQ_MULTI_OF = 32
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class UnifiedPrepare(nn.Module):
|
||||
"""Prepares unified tensors for transformer blocks.
|
||||
|
||||
This module encapsulates the unification of x and cap tensors into unified
|
||||
sequences. Similar to how Wan's `rope` module outputs rotary embeddings,
|
||||
this module outputs unified tensors that can be sharded via _sp_plan's
|
||||
split_output=True mechanism.
|
||||
|
||||
This follows the diffusers pattern where tensor preparation happens in
|
||||
a dedicated submodule, enabling _sp_plan hooks to work at module boundaries.
|
||||
"""
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
x_cos: torch.Tensor,
|
||||
x_sin: torch.Tensor,
|
||||
cap_feats: torch.Tensor,
|
||||
cap_cos: torch.Tensor,
|
||||
cap_sin: torch.Tensor,
|
||||
x_item_seqlens: list[int],
|
||||
cap_item_seqlens: list[int],
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Combine x and cap tensors into unified sequences.
|
||||
|
||||
Returns:
|
||||
unified: Combined hidden states [batch, seq_len, dim]
|
||||
unified_cos: Combined RoPE cos [batch, seq_len, rope_dim]
|
||||
unified_sin: Combined RoPE sin [batch, seq_len, rope_dim]
|
||||
unified_attn_mask: Combined attention mask [batch, seq_len]
|
||||
"""
|
||||
bsz = x.shape[0]
|
||||
device = x.device
|
||||
|
||||
unified = []
|
||||
unified_cos = []
|
||||
unified_sin = []
|
||||
for i in range(bsz):
|
||||
x_len = x_item_seqlens[i]
|
||||
cap_len = cap_item_seqlens[i]
|
||||
unified.append(torch.cat([x[i][:x_len], cap_feats[i][:cap_len]]))
|
||||
unified_cos.append(torch.cat([x_cos[i][:x_len], cap_cos[i][:cap_len]]))
|
||||
unified_sin.append(torch.cat([x_sin[i][:x_len], cap_sin[i][:cap_len]]))
|
||||
|
||||
unified_item_seqlens = [a + b for a, b in zip(cap_item_seqlens, x_item_seqlens)]
|
||||
unified_max_item_seqlen = max(unified_item_seqlens)
|
||||
|
||||
unified = pad_sequence(unified, batch_first=True, padding_value=0.0)
|
||||
unified_cos = pad_sequence(unified_cos, batch_first=True, padding_value=0.0)
|
||||
unified_sin = pad_sequence(unified_sin, batch_first=True, padding_value=0.0)
|
||||
unified_attn_mask = torch.zeros((bsz, unified_max_item_seqlen), dtype=torch.bool, device=device)
|
||||
for i, seq_len in enumerate(unified_item_seqlens):
|
||||
unified_attn_mask[i, :seq_len] = 1
|
||||
|
||||
return unified, unified_cos, unified_sin, unified_attn_mask
|
||||
|
||||
|
||||
def _positive_divisors(n: int) -> set[int]:
|
||||
if n <= 0:
|
||||
return set()
|
||||
@@ -456,8 +521,52 @@ class RopeEmbedder:
|
||||
|
||||
|
||||
class ZImageTransformer2DModel(nn.Module):
|
||||
"""Z-Image Transformer model for image generation.
|
||||
|
||||
Sequence Parallelism:
|
||||
This model supports non-intrusive SP via _sp_plan. The plan specifies:
|
||||
- Input splitting at first main transformer block (unified sequence)
|
||||
- RoPE (cos/sin) splitting along sequence dimension
|
||||
- Attention mask splitting along sequence dimension
|
||||
- Output gathering at final_layer
|
||||
|
||||
The SP is applied to the main `layers` transformer blocks where the
|
||||
unified image+caption sequence is processed jointly.
|
||||
|
||||
Note: noise_refiner and context_refiner are NOT parallelized as they
|
||||
process image and caption separately before unification.
|
||||
|
||||
Important: The default _sp_plan assumes patch_size=2 and f_patch_size=1.
|
||||
If using different patch configurations, update _sp_plan accordingly.
|
||||
|
||||
Note: Our "Sequence Parallelism" (SP) corresponds to "Context Parallelism" (CP) in diffusers.
|
||||
"""
|
||||
|
||||
_repeated_blocks = ["ZImageTransformerBlock"]
|
||||
|
||||
# Sequence Parallelism for Z-Image (following diffusers' _cp_plan pattern)
|
||||
# Similar to how Wan uses `rope` module's split_output to shard rotary embeddings,
|
||||
# Z-Image uses `unified_prepare` module's split_output to shard unified tensors.
|
||||
#
|
||||
# The _sp_plan specifies sharding/gathering at module boundaries:
|
||||
# - unified_prepare: Split all 4 outputs (unified, cos, sin, attn_mask) via split_output=True
|
||||
# - layers.0: hidden_states input is already sharded from unified_prepare output
|
||||
# - all_final_layer.2-1: Gather outputs after the final layer
|
||||
#
|
||||
# Note: _sp_plan corresponds to diffusers' _cp_plan (Context Parallelism)
|
||||
_sp_plan = {
|
||||
# Shard unified_prepare outputs (similar to Wan's rope module)
|
||||
# This shards all 4 return values: unified, unified_cos, unified_sin, unified_attn_mask
|
||||
"unified_prepare": {
|
||||
0: SequenceParallelInput(split_dim=1, expected_dims=3, split_output=True), # unified
|
||||
1: SequenceParallelInput(split_dim=1, expected_dims=3, split_output=True), # unified_cos
|
||||
2: SequenceParallelInput(split_dim=1, expected_dims=3, split_output=True), # unified_sin
|
||||
3: SequenceParallelInput(split_dim=1, expected_dims=2, split_output=True), # unified_attn_mask
|
||||
},
|
||||
# Gather output at final_layer (default: patch_size=2, f_patch_size=1)
|
||||
"all_final_layer.2-1": SequenceParallelOutput(gather_dim=1, expected_dims=3),
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
all_patch_size=(2,),
|
||||
@@ -572,6 +681,11 @@ class ZImageTransformer2DModel(nn.Module):
|
||||
|
||||
self.rope_embedder = RopeEmbedder(theta=rope_theta, axes_dims=axes_dims, axes_lens=axes_lens)
|
||||
|
||||
# UnifiedPrepare module for combining x and cap tensors
|
||||
# This enables _cp_plan to shard outputs via split_output=True
|
||||
# Similar to how Wan's rope module enables rotary embedding sharding
|
||||
self.unified_prepare = UnifiedPrepare()
|
||||
|
||||
def unpatchify(self, x: list[torch.Tensor], size: list[tuple], patch_size, f_patch_size) -> list[torch.Tensor]:
|
||||
pH = pW = patch_size
|
||||
pF = f_patch_size
|
||||
@@ -772,31 +886,19 @@ class ZImageTransformer2DModel(nn.Module):
|
||||
for layer in self.context_refiner:
|
||||
cap_feats = layer(cap_feats, cap_attn_mask, cap_cos, cap_sin)
|
||||
|
||||
# unified
|
||||
unified = []
|
||||
unified_cos = []
|
||||
unified_sin = []
|
||||
for i in range(bsz):
|
||||
x_len = x_item_seqlens[i]
|
||||
cap_len = cap_item_seqlens[i]
|
||||
unified.append(torch.cat([x[i][:x_len], cap_feats[i][:cap_len]]))
|
||||
unified_cos.append(torch.cat([x_cos[i][:x_len], cap_cos[i][:cap_len]]))
|
||||
unified_sin.append(torch.cat([x_sin[i][:x_len], cap_sin[i][:cap_len]]))
|
||||
unified_item_seqlens = [a + b for a, b in zip(cap_item_seqlens, x_item_seqlens)]
|
||||
assert unified_item_seqlens == [len(_) for _ in unified]
|
||||
unified_max_item_seqlen = max(unified_item_seqlens)
|
||||
|
||||
unified = pad_sequence(unified, batch_first=True, padding_value=0.0)
|
||||
unified_cos = pad_sequence(unified_cos, batch_first=True, padding_value=0.0)
|
||||
unified_sin = pad_sequence(unified_sin, batch_first=True, padding_value=0.0)
|
||||
unified_attn_mask = torch.zeros((bsz, unified_max_item_seqlen), dtype=torch.bool, device=device)
|
||||
for i, seq_len in enumerate(unified_item_seqlens):
|
||||
unified_attn_mask[i, :seq_len] = 1
|
||||
# Prepare unified tensors via UnifiedPrepare module
|
||||
# This enables _cp_plan to shard outputs via split_output=True
|
||||
unified, unified_cos, unified_sin, unified_attn_mask = self.unified_prepare(
|
||||
x, x_cos, x_sin, cap_feats, cap_cos, cap_sin, x_item_seqlens, cap_item_seqlens
|
||||
)
|
||||
|
||||
# Main transformer blocks
|
||||
for layer in self.layers:
|
||||
unified = layer(unified, unified_attn_mask, unified_cos, unified_sin, adaln_input)
|
||||
|
||||
# Final layer
|
||||
unified = self.all_final_layer[f"{patch_size}-{f_patch_size}"](unified, adaln_input)
|
||||
|
||||
unified = list(unified.unbind(dim=0))
|
||||
x = self.unpatchify(unified, x_size, patch_size, f_patch_size)
|
||||
|
||||
|
||||
Reference in new issue
Block a user