[Model] Add LingBot World Ulysses sequence parallelism (#6841)
Signed-off-by: wtz2333 <2955110911@qq.com> Co-authored-by: Zhou Taichang <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
de9a133d48
commit
02aaa34059
8 files changed
+659
-45
No files matched your search
@@ -18,7 +18,6 @@ import torch.nn.functional as F
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
from torch import nn
|
||||
from transformers import AutoTokenizer, UMT5EncoderModel
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
@@ -203,7 +202,6 @@ def _validate_parallel_config(od_config: OmniDiffusionConfig) -> None:
|
||||
return
|
||||
unsupported_sizes = {
|
||||
"pipeline_parallel_size": "pipeline parallelism",
|
||||
"sequence_parallel_size": "sequence parallelism",
|
||||
"cfg_parallel_size": "CFG parallelism",
|
||||
"vae_patch_parallel_size": "VAE parallelism",
|
||||
}
|
||||
@@ -211,6 +209,25 @@ def _validate_parallel_config(od_config: OmniDiffusionConfig) -> None:
|
||||
size = getattr(parallel_config, field, 1) or 1
|
||||
if size > 1:
|
||||
raise NotImplementedError(f"LingBot World v1 does not support {feature} ({field}={size}).")
|
||||
sequence_parallel_size = getattr(parallel_config, "sequence_parallel_size", 1) or 1
|
||||
ulysses_degree = getattr(parallel_config, "ulysses_degree", 1) or 1
|
||||
ring_degree = getattr(parallel_config, "ring_degree", 1) or 1
|
||||
allgather_degree = getattr(parallel_config, "allgather_degree", 1) or 1
|
||||
ulysses_mode = getattr(parallel_config, "ulysses_mode", "strict")
|
||||
ulysses_a2a_permute = bool(getattr(parallel_config, "ulysses_a2a_permute", False))
|
||||
if (
|
||||
sequence_parallel_size != ulysses_degree
|
||||
or ring_degree != 1
|
||||
or allgather_degree != 1
|
||||
or ulysses_mode != "strict"
|
||||
or ulysses_a2a_permute
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"LingBot World sequence parallelism requires pure Ulysses with ulysses_a2a_permute disabled: "
|
||||
f"sequence_parallel_size={sequence_parallel_size}, ulysses_degree={ulysses_degree}, "
|
||||
f"ring_degree={ring_degree}, allgather_degree={allgather_degree}, ulysses_mode={ulysses_mode!r}, "
|
||||
f"ulysses_a2a_permute={ulysses_a2a_permute}."
|
||||
)
|
||||
if getattr(parallel_config, "use_hsdp", False):
|
||||
raise NotImplementedError("LingBot World v1 does not support HSDP.")
|
||||
if getattr(parallel_config, "enable_expert_parallel", False):
|
||||
@@ -557,8 +574,6 @@ class LingBotWorldCausalDMDPipeline(
|
||||
latent_height = self._ar_height // spatial
|
||||
latent_width = self._ar_width // spatial
|
||||
tokens_per_frame = (latent_height // patch_height) * (latent_width // patch_width)
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
num_local_heads = int(self.transformer.config.num_attention_heads) // tp_size
|
||||
total_window_frames = (
|
||||
int(self.transformer.config.local_attn_size)
|
||||
if int(self.transformer.config.local_attn_size) != -1
|
||||
@@ -579,7 +594,7 @@ class LingBotWorldCausalDMDPipeline(
|
||||
)
|
||||
return ARDiffusionKVCacheSpec(
|
||||
num_layers=int(self.transformer.config.num_layers),
|
||||
num_kv_heads=num_local_heads,
|
||||
num_kv_heads=int(self.transformer.blocks[0].self_attn.num_sp_heads),
|
||||
head_size=int(self.transformer.config.attention_head_dim),
|
||||
tokens_per_frame=tokens_per_frame,
|
||||
frames_per_block=int(self.transformer.config.num_frames_per_block),
|
||||
@@ -960,20 +975,11 @@ class LingBotWorldCausalDMDPipeline(
|
||||
def layer_kv() -> Iterator[tuple[torch.Tensor, torch.Tensor]]:
|
||||
for block in self.transformer.blocks:
|
||||
cross_attention = block.cross_attn
|
||||
key = cross_attention.norm_k(cross_attention.k(projected_text)).unflatten(
|
||||
2,
|
||||
(
|
||||
cross_attention.num_local_heads,
|
||||
cross_attention.head_dim,
|
||||
),
|
||||
)
|
||||
value = cross_attention.v(projected_text).unflatten(
|
||||
2,
|
||||
(
|
||||
cross_attention.num_local_heads,
|
||||
cross_attention.head_dim,
|
||||
),
|
||||
shape = (cross_attention.num_local_heads, cross_attention.head_dim)
|
||||
key = cross_attention.shard_kv_heads(
|
||||
cross_attention.norm_k(cross_attention.k(projected_text)).unflatten(2, shape)
|
||||
)
|
||||
value = cross_attention.shard_kv_heads(cross_attention.v(projected_text).unflatten(2, shape))
|
||||
yield key, value
|
||||
|
||||
state.populate_cross_attention(
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Checkpoint-compatible causal DiT for the LingBot World v2 model package."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -24,6 +24,9 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.distributed.comm import SeqAllToAll4D
|
||||
from vllm_omni.diffusion.distributed.parallel_state import get_sp_group
|
||||
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelInput, SequenceParallelOutput
|
||||
from vllm_omni.diffusion.layers.norm import LayerNorm
|
||||
from vllm_omni.diffusion.layers.rope import RotaryEmbeddingWan
|
||||
from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import (
|
||||
@@ -127,6 +130,37 @@ def _projection_prefix(prefix: str, name: str) -> str:
|
||||
return f"{prefix}.{name}" if prefix else name
|
||||
|
||||
|
||||
def _ulysses_state() -> tuple[int, int, torch.distributed.ProcessGroup | None]:
|
||||
coordinator = get_sp_group()
|
||||
return (
|
||||
int(coordinator.ulysses_world_size),
|
||||
int(coordinator.ulysses_rank),
|
||||
coordinator.ulysses_group,
|
||||
)
|
||||
|
||||
|
||||
class _LingBotSPPrepare(nn.Module):
|
||||
"""Expand frame conditioning only when Ulysses shards the token sequence."""
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
camera_hidden_states: torch.Tensor,
|
||||
timestep_projection: torch.Tensor,
|
||||
rotary_emb: tuple[torch.Tensor, torch.Tensor],
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
num_frames = timestep_projection.shape[1]
|
||||
if hidden_states.shape[1] % num_frames:
|
||||
raise ValueError("LingBot token count must be divisible by the latent frame count.")
|
||||
tokens_per_frame = hidden_states.shape[1] // num_frames
|
||||
if get_sp_group().ulysses_world_size > 1:
|
||||
timestep_projection = (
|
||||
timestep_projection.unsqueeze(2).expand(-1, -1, tokens_per_frame, -1, -1).flatten(1, 2)
|
||||
)
|
||||
cosine, sine = rotary_emb
|
||||
return hidden_states, camera_hidden_states, timestep_projection, cosine, sine
|
||||
|
||||
|
||||
class LingBotSelfAttention(nn.Module):
|
||||
"""Block-causal self-attention over retained history and one full chunk."""
|
||||
|
||||
@@ -157,6 +191,13 @@ class LingBotSelfAttention(nn.Module):
|
||||
prefix=_projection_prefix(prefix, "qkv"),
|
||||
)
|
||||
self.num_local_heads = self.qkv.num_heads
|
||||
self.ulysses_world_size, self.ulysses_rank, self.ulysses_group = _ulysses_state()
|
||||
if self.num_local_heads % self.ulysses_world_size:
|
||||
raise ValueError(
|
||||
"LingBot local attention heads must be divisible by the Ulysses degree: "
|
||||
f"heads={self.num_local_heads}, ulysses={self.ulysses_world_size}."
|
||||
)
|
||||
self.num_sp_heads = self.num_local_heads // self.ulysses_world_size
|
||||
self.tp_inner_dim = self.num_local_heads * self.head_dim
|
||||
self.o = RowParallelLinear(
|
||||
dim,
|
||||
@@ -170,9 +211,9 @@ class LingBotSelfAttention(nn.Module):
|
||||
self.norm_k = _LingBotRMSNorm(self.tp_inner_dim, eps)
|
||||
self.rotary_embedding = RotaryEmbeddingWan(is_neox_style=False, half_head_dim=True)
|
||||
self.attn = Attention(
|
||||
num_heads=self.num_local_heads,
|
||||
num_heads=self.num_sp_heads,
|
||||
head_size=self.head_dim,
|
||||
num_kv_heads=self.num_local_heads,
|
||||
num_kv_heads=self.num_sp_heads,
|
||||
softmax_scale=self.head_dim**-0.5,
|
||||
causal=False,
|
||||
role="self",
|
||||
@@ -287,6 +328,11 @@ class LingBotSelfAttention(nn.Module):
|
||||
query = self.rotary_embedding(query, cos, sin)
|
||||
key = self.rotary_embedding(key, cos, sin)
|
||||
|
||||
if self.ulysses_world_size > 1:
|
||||
query = SeqAllToAll4D.apply(self.ulysses_group, query, 2, 1, False)
|
||||
key = SeqAllToAll4D.apply(self.ulysses_group, key, 2, 1, False)
|
||||
value = SeqAllToAll4D.apply(self.ulysses_group, value, 2, 1, False)
|
||||
|
||||
if isinstance(cache, ARDiffusionPagedLayerInputs):
|
||||
if query.shape[0] != 1:
|
||||
raise RuntimeError("LingBot AR-Diffusion paged attention requires batch_size=1.")
|
||||
@@ -347,6 +393,8 @@ class LingBotSelfAttention(nn.Module):
|
||||
)
|
||||
else:
|
||||
output = self.attn(query, visible_key, visible_value)
|
||||
if self.ulysses_world_size > 1:
|
||||
output = SeqAllToAll4D.apply(self.ulysses_group, output, 1, 2, False)
|
||||
return self.o(output.flatten(2, 3))
|
||||
|
||||
|
||||
@@ -373,6 +421,13 @@ class LingBotCrossAttention(nn.Module):
|
||||
self.num_heads = num_heads
|
||||
self.head_dim = dim // num_heads
|
||||
self.num_local_heads = num_heads // tp_size
|
||||
self.ulysses_world_size, self.ulysses_rank, self.ulysses_group = _ulysses_state()
|
||||
if self.num_local_heads % self.ulysses_world_size:
|
||||
raise ValueError(
|
||||
"LingBot local cross-attention heads must be divisible by the Ulysses degree: "
|
||||
f"heads={self.num_local_heads}, ulysses={self.ulysses_world_size}."
|
||||
)
|
||||
self.num_sp_heads = self.num_local_heads // self.ulysses_world_size
|
||||
self.tp_inner_dim = self.num_local_heads * self.head_dim
|
||||
|
||||
self.q = ColumnParallelLinear(
|
||||
@@ -410,9 +465,9 @@ class LingBotCrossAttention(nn.Module):
|
||||
self.norm_q = _LingBotRMSNorm(self.tp_inner_dim, eps)
|
||||
self.norm_k = _LingBotRMSNorm(self.tp_inner_dim, eps)
|
||||
self.attn = Attention(
|
||||
num_heads=self.num_local_heads,
|
||||
num_heads=self.num_sp_heads,
|
||||
head_size=self.head_dim,
|
||||
num_kv_heads=self.num_local_heads,
|
||||
num_kv_heads=self.num_sp_heads,
|
||||
softmax_scale=self.head_dim**-0.5,
|
||||
causal=False,
|
||||
role="cross",
|
||||
@@ -422,6 +477,14 @@ class LingBotCrossAttention(nn.Module):
|
||||
disable_kv_quant=True,
|
||||
)
|
||||
|
||||
def shard_kv_heads(self, value: torch.Tensor) -> torch.Tensor:
|
||||
"""Select this Ulysses rank's projected K/V heads."""
|
||||
|
||||
if self.ulysses_world_size == 1:
|
||||
return value
|
||||
start = self.ulysses_rank * self.num_sp_heads
|
||||
return value[:, :, start : start + self.num_sp_heads].clone(memory_format=torch.contiguous_format)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -431,6 +494,8 @@ class LingBotCrossAttention(nn.Module):
|
||||
) -> tuple[torch.Tensor, LingBotAttentionCache]:
|
||||
query = self.norm_q(self.q(hidden_states))
|
||||
query = query.unflatten(2, (self.num_local_heads, self.head_dim))
|
||||
if self.ulysses_world_size > 1:
|
||||
query = SeqAllToAll4D.apply(self.ulysses_group, query, 2, 1, False)
|
||||
|
||||
# Text K/V is constant within a request and is projected once per layer.
|
||||
if cache is None:
|
||||
@@ -438,8 +503,8 @@ class LingBotCrossAttention(nn.Module):
|
||||
raise ValueError("encoder_hidden_states are required when the cross-attention cache is empty.")
|
||||
key = self.norm_k(self.k(encoder_hidden_states))
|
||||
value = self.v(encoder_hidden_states)
|
||||
key = key.unflatten(2, (self.num_local_heads, self.head_dim))
|
||||
value = value.unflatten(2, (self.num_local_heads, self.head_dim))
|
||||
key = self.shard_kv_heads(key.unflatten(2, (self.num_local_heads, self.head_dim)))
|
||||
value = self.shard_kv_heads(value.unflatten(2, (self.num_local_heads, self.head_dim)))
|
||||
cache = LingBotAttentionCache(
|
||||
key=key,
|
||||
value=value,
|
||||
@@ -452,6 +517,8 @@ class LingBotCrossAttention(nn.Module):
|
||||
value = cache.value[:, : cache.end]
|
||||
|
||||
output = self.attn(query, key, value)
|
||||
if self.ulysses_world_size > 1:
|
||||
output = SeqAllToAll4D.apply(self.ulysses_group, output, 1, 2, False)
|
||||
return self.o(output.flatten(2, 3)), cache
|
||||
|
||||
|
||||
@@ -540,7 +607,18 @@ class LingBotAttentionBlock(nn.Module):
|
||||
rotary_emb: tuple[torch.Tensor, torch.Tensor],
|
||||
) -> tuple[torch.Tensor, LingBotAttentionCache]:
|
||||
batch_size, token_count, dim = hidden_states.shape
|
||||
if (
|
||||
timestep_projection.ndim != 4
|
||||
or timestep_projection.shape[0] != batch_size
|
||||
or timestep_projection.shape[2:] != (6, dim)
|
||||
):
|
||||
raise ValueError("LingBot timestep projection must have shape (batch, frames or tokens, 6, dim).")
|
||||
num_frames = timestep_projection.shape[1]
|
||||
if num_frames == 0 or token_count % num_frames:
|
||||
raise ValueError("LingBot timestep projection must align with hidden tokens.")
|
||||
if self.self_attn.ulysses_world_size > 1 and num_frames != token_count:
|
||||
raise ValueError("LingBot Ulysses requires one timestep projection per local token.")
|
||||
# Without SP, broadcast per frame. With SP, each entry represents one token.
|
||||
tokens_per_frame = token_count // num_frames
|
||||
# Timestep, camera, and text remain separate conditioning paths.
|
||||
modulation = self.modulation.unsqueeze(1) + timestep_projection.float()
|
||||
@@ -635,13 +713,26 @@ class _LingBotHead(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
timestep_embedding: torch.Tensor,
|
||||
*,
|
||||
tokens_per_frame: int | None = None,
|
||||
token_offset: int = 0,
|
||||
) -> torch.Tensor:
|
||||
num_frames = timestep_embedding.shape[1]
|
||||
tokens_per_frame = hidden_states.shape[1] // num_frames
|
||||
modulation = self.modulation.unsqueeze(1) + timestep_embedding.unsqueeze(2).float()
|
||||
shift, scale = modulation.chunk(2, dim=2)
|
||||
normalized = self.norm(hidden_states.float()).unflatten(1, (num_frames, tokens_per_frame))
|
||||
normalized = (normalized * (1 + scale) + shift).flatten(1, 2).to(hidden_states.dtype)
|
||||
normalized = self.norm(hidden_states.float())
|
||||
if tokens_per_frame is None:
|
||||
normalized = normalized.unflatten(1, (num_frames, -1))
|
||||
normalized = (normalized * (1 + scale) + shift).flatten(1, 2)
|
||||
else:
|
||||
# A sequence shard can begin/end inside a frame.
|
||||
frames = (
|
||||
torch.arange(hidden_states.shape[1], device=hidden_states.device) + token_offset
|
||||
) // tokens_per_frame
|
||||
scale = scale.squeeze(2).index_select(1, frames)
|
||||
shift = shift.squeeze(2).index_select(1, frames)
|
||||
normalized = normalized * (1 + scale) + shift
|
||||
normalized = normalized.to(hidden_states.dtype)
|
||||
return self.head(normalized)
|
||||
|
||||
|
||||
@@ -678,6 +769,16 @@ class CausalLingBotWorldTransformer3DModel(nn.Module):
|
||||
_repeated_blocks = ["LingBotAttentionBlock"]
|
||||
packed_modules_mapping = {"qkv": ["q", "k", "v"]}
|
||||
_layerwise_offload_blocks_attrs = ["blocks"]
|
||||
_sp_plan = {
|
||||
"sp_prepare": {
|
||||
0: SequenceParallelInput(split_dim=1, expected_dims=3, split_output=True),
|
||||
1: SequenceParallelInput(split_dim=1, expected_dims=3, split_output=True),
|
||||
2: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True),
|
||||
3: SequenceParallelInput(split_dim=0, expected_dims=2, split_output=True),
|
||||
4: SequenceParallelInput(split_dim=0, expected_dims=2, split_output=True),
|
||||
},
|
||||
"sp_output_gather": SequenceParallelOutput(gather_dim=1, expected_dims=3),
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -768,6 +869,8 @@ class CausalLingBotWorldTransformer3DModel(nn.Module):
|
||||
stride=patch_size,
|
||||
)
|
||||
self.patch_embedding_wancamctrl = _LingBotCameraPatchEmbedding(6 * 8 * 8, dim, patch_size)
|
||||
self.sp_prepare = _LingBotSPPrepare()
|
||||
self.sp_output_gather = nn.Identity()
|
||||
self.c2ws_hidden_states_layer1 = ColumnParallelLinear(
|
||||
dim,
|
||||
dim,
|
||||
@@ -904,8 +1007,7 @@ class CausalLingBotWorldTransformer3DModel(nn.Module):
|
||||
self.config.local_attn_size if self.config.local_attn_size != -1 else self.config.sliding_window_num_frames
|
||||
)
|
||||
max_tokens = int(window_frames * post_patch_height * post_patch_width)
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
num_local_heads = self.config.num_attention_heads // tp_size
|
||||
num_local_heads = self.blocks[0].self_attn.num_sp_heads
|
||||
return allocate_lingbot_cache(
|
||||
batch_size=batch_size,
|
||||
num_layers=self.config.num_layers,
|
||||
@@ -1138,6 +1240,33 @@ class CausalLingBotWorldTransformer3DModel(nn.Module):
|
||||
query_len=hidden_states.shape[1],
|
||||
)
|
||||
cache.self_attention = [layer_context.to_layer_inputs() for layer_context in cache.self_attention]
|
||||
sp_size = self.blocks[0].self_attn.ulysses_world_size
|
||||
global_tokens = patched_frames * tokens_per_frame
|
||||
hidden_dim = hidden_states.shape[-1]
|
||||
rope_dim = rotary_emb[0].shape[-1]
|
||||
if global_tokens % sp_size:
|
||||
raise ValueError("LingBot Ulysses requires the token count to be divisible by its degree.")
|
||||
hidden_states, camera_hidden_states, timestep_projection, cosine, sine = self.sp_prepare(
|
||||
hidden_states,
|
||||
camera_hidden_states,
|
||||
timestep_projection,
|
||||
rotary_emb,
|
||||
)
|
||||
if sp_size > 1:
|
||||
local_tokens = global_tokens // sp_size
|
||||
expected_hidden = (batch_size, local_tokens, hidden_dim)
|
||||
if (
|
||||
hidden_states.shape != expected_hidden
|
||||
or camera_hidden_states.shape != expected_hidden
|
||||
or timestep_projection.shape != (batch_size, local_tokens, 6, hidden_dim)
|
||||
or cosine.shape != (local_tokens, rope_dim)
|
||||
or sine.shape != (local_tokens, rope_dim)
|
||||
):
|
||||
raise RuntimeError(
|
||||
"LingBot SP input hooks did not shard all conditioning tensors consistently; "
|
||||
f"expected {local_tokens} tokens per rank. Check SP hook registration and input dimensions."
|
||||
)
|
||||
rotary_emb = (cosine, sine)
|
||||
# Phase 3: each layer receives its own cache entry. Text K/V is passed
|
||||
# only when absent; the returned cache is stored for subsequent DMD
|
||||
# steps and causal blocks in this request.
|
||||
@@ -1156,9 +1285,17 @@ class CausalLingBotWorldTransformer3DModel(nn.Module):
|
||||
)
|
||||
cache.cross_attention[index] = cross_cache
|
||||
|
||||
# Phase 4: map tokens to per-patch 16-channel flow values and restore
|
||||
# [B, C, F, H, W] for the Pipeline's sampler update.
|
||||
hidden_states = self.head(hidden_states, timestep_embedding)
|
||||
# Phase 4: project local tokens before gathering the much narrower flow values.
|
||||
hidden_states = self.head(
|
||||
hidden_states,
|
||||
timestep_embedding,
|
||||
tokens_per_frame=tokens_per_frame if sp_size > 1 else None,
|
||||
token_offset=get_sp_group().ulysses_rank * (global_tokens // sp_size) if sp_size > 1 else 0,
|
||||
)
|
||||
hidden_states = self.sp_output_gather(hidden_states)
|
||||
projected_dim = self.config.out_channels * math.prod(self.config.patch_size)
|
||||
if sp_size > 1 and hidden_states.shape != (batch_size, global_tokens, projected_dim):
|
||||
raise RuntimeError("LingBot SP output hook did not gather the full projected token sequence.")
|
||||
return self._unpatchify(
|
||||
hidden_states,
|
||||
batch_size=batch_size,
|
||||
|
||||
Reference in new issue
Block a user