[Model] Add LingBot World Ulysses sequence parallelism (#6841)

Signed-off-by: wtz2333 <2955110911@qq.com>
Co-authored-by: Zhou Taichang <tzhouam@connect.ust.hk>
This commit is contained in:
wtz2333andZhou Taichang authored and GitHub committed 2026-09-11 18:37:12 +08:00
1 parent de9a133d48
commit 02aaa34059
8 files changed
+659 -45

No files matched your search

@@ -18,7 +18,6 @@ import torch.nn.functional as F
from diffusers.utils.torch_utils import randn_tensor
from torch import nn
from transformers import AutoTokenizer, UMT5EncoderModel
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
@@ -203,7 +202,6 @@ def _validate_parallel_config(od_config: OmniDiffusionConfig) -> None:
return
unsupported_sizes = {
"pipeline_parallel_size": "pipeline parallelism",
"sequence_parallel_size": "sequence parallelism",
"cfg_parallel_size": "CFG parallelism",
"vae_patch_parallel_size": "VAE parallelism",
}
@@ -211,6 +209,25 @@ def _validate_parallel_config(od_config: OmniDiffusionConfig) -> None:
size = getattr(parallel_config, field, 1) or 1
if size > 1:
raise NotImplementedError(f"LingBot World v1 does not support {feature} ({field}={size}).")
sequence_parallel_size = getattr(parallel_config, "sequence_parallel_size", 1) or 1
ulysses_degree = getattr(parallel_config, "ulysses_degree", 1) or 1
ring_degree = getattr(parallel_config, "ring_degree", 1) or 1
allgather_degree = getattr(parallel_config, "allgather_degree", 1) or 1
ulysses_mode = getattr(parallel_config, "ulysses_mode", "strict")
ulysses_a2a_permute = bool(getattr(parallel_config, "ulysses_a2a_permute", False))
if (
sequence_parallel_size != ulysses_degree
or ring_degree != 1
or allgather_degree != 1
or ulysses_mode != "strict"
or ulysses_a2a_permute
):
raise NotImplementedError(
"LingBot World sequence parallelism requires pure Ulysses with ulysses_a2a_permute disabled: "
f"sequence_parallel_size={sequence_parallel_size}, ulysses_degree={ulysses_degree}, "
f"ring_degree={ring_degree}, allgather_degree={allgather_degree}, ulysses_mode={ulysses_mode!r}, "
f"ulysses_a2a_permute={ulysses_a2a_permute}."
)
if getattr(parallel_config, "use_hsdp", False):
raise NotImplementedError("LingBot World v1 does not support HSDP.")
if getattr(parallel_config, "enable_expert_parallel", False):
@@ -557,8 +574,6 @@ class LingBotWorldCausalDMDPipeline(
latent_height = self._ar_height // spatial
latent_width = self._ar_width // spatial
tokens_per_frame = (latent_height // patch_height) * (latent_width // patch_width)
tp_size = get_tensor_model_parallel_world_size()
num_local_heads = int(self.transformer.config.num_attention_heads) // tp_size
total_window_frames = (
int(self.transformer.config.local_attn_size)
if int(self.transformer.config.local_attn_size) != -1
@@ -579,7 +594,7 @@ class LingBotWorldCausalDMDPipeline(
)
return ARDiffusionKVCacheSpec(
num_layers=int(self.transformer.config.num_layers),
num_kv_heads=num_local_heads,
num_kv_heads=int(self.transformer.blocks[0].self_attn.num_sp_heads),
head_size=int(self.transformer.config.attention_head_dim),
tokens_per_frame=tokens_per_frame,
frames_per_block=int(self.transformer.config.num_frames_per_block),
@@ -960,20 +975,11 @@ class LingBotWorldCausalDMDPipeline(
def layer_kv() -> Iterator[tuple[torch.Tensor, torch.Tensor]]:
for block in self.transformer.blocks:
cross_attention = block.cross_attn
key = cross_attention.norm_k(cross_attention.k(projected_text)).unflatten(
2,
(
cross_attention.num_local_heads,
cross_attention.head_dim,
),
)
value = cross_attention.v(projected_text).unflatten(
2,
(
cross_attention.num_local_heads,
cross_attention.head_dim,
),
shape = (cross_attention.num_local_heads, cross_attention.head_dim)
key = cross_attention.shard_kv_heads(
cross_attention.norm_k(cross_attention.k(projected_text)).unflatten(2, shape)
)
value = cross_attention.shard_kv_heads(cross_attention.v(projected_text).unflatten(2, shape))
yield key, value
state.populate_cross_attention(
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Checkpoint-compatible causal DiT for the LingBot World v2 model package."""
from __future__ import annotations
@@ -24,6 +24,9 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.distributed.comm import SeqAllToAll4D
from vllm_omni.diffusion.distributed.parallel_state import get_sp_group
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelInput, SequenceParallelOutput
from vllm_omni.diffusion.layers.norm import LayerNorm
from vllm_omni.diffusion.layers.rope import RotaryEmbeddingWan
from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import (
@@ -127,6 +130,37 @@ def _projection_prefix(prefix: str, name: str) -> str:
return f"{prefix}.{name}" if prefix else name
def _ulysses_state() -> tuple[int, int, torch.distributed.ProcessGroup | None]:
coordinator = get_sp_group()
return (
int(coordinator.ulysses_world_size),
int(coordinator.ulysses_rank),
coordinator.ulysses_group,
)
class _LingBotSPPrepare(nn.Module):
"""Expand frame conditioning only when Ulysses shards the token sequence."""
def forward(
self,
hidden_states: torch.Tensor,
camera_hidden_states: torch.Tensor,
timestep_projection: torch.Tensor,
rotary_emb: tuple[torch.Tensor, torch.Tensor],
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
num_frames = timestep_projection.shape[1]
if hidden_states.shape[1] % num_frames:
raise ValueError("LingBot token count must be divisible by the latent frame count.")
tokens_per_frame = hidden_states.shape[1] // num_frames
if get_sp_group().ulysses_world_size > 1:
timestep_projection = (
timestep_projection.unsqueeze(2).expand(-1, -1, tokens_per_frame, -1, -1).flatten(1, 2)
)
cosine, sine = rotary_emb
return hidden_states, camera_hidden_states, timestep_projection, cosine, sine
class LingBotSelfAttention(nn.Module):
"""Block-causal self-attention over retained history and one full chunk."""
@@ -157,6 +191,13 @@ class LingBotSelfAttention(nn.Module):
prefix=_projection_prefix(prefix, "qkv"),
)
self.num_local_heads = self.qkv.num_heads
self.ulysses_world_size, self.ulysses_rank, self.ulysses_group = _ulysses_state()
if self.num_local_heads % self.ulysses_world_size:
raise ValueError(
"LingBot local attention heads must be divisible by the Ulysses degree: "
f"heads={self.num_local_heads}, ulysses={self.ulysses_world_size}."
)
self.num_sp_heads = self.num_local_heads // self.ulysses_world_size
self.tp_inner_dim = self.num_local_heads * self.head_dim
self.o = RowParallelLinear(
dim,
@@ -170,9 +211,9 @@ class LingBotSelfAttention(nn.Module):
self.norm_k = _LingBotRMSNorm(self.tp_inner_dim, eps)
self.rotary_embedding = RotaryEmbeddingWan(is_neox_style=False, half_head_dim=True)
self.attn = Attention(
num_heads=self.num_local_heads,
num_heads=self.num_sp_heads,
head_size=self.head_dim,
num_kv_heads=self.num_local_heads,
num_kv_heads=self.num_sp_heads,
softmax_scale=self.head_dim**-0.5,
causal=False,
role="self",
@@ -287,6 +328,11 @@ class LingBotSelfAttention(nn.Module):
query = self.rotary_embedding(query, cos, sin)
key = self.rotary_embedding(key, cos, sin)
if self.ulysses_world_size > 1:
query = SeqAllToAll4D.apply(self.ulysses_group, query, 2, 1, False)
key = SeqAllToAll4D.apply(self.ulysses_group, key, 2, 1, False)
value = SeqAllToAll4D.apply(self.ulysses_group, value, 2, 1, False)
if isinstance(cache, ARDiffusionPagedLayerInputs):
if query.shape[0] != 1:
raise RuntimeError("LingBot AR-Diffusion paged attention requires batch_size=1.")
@@ -347,6 +393,8 @@ class LingBotSelfAttention(nn.Module):
)
else:
output = self.attn(query, visible_key, visible_value)
if self.ulysses_world_size > 1:
output = SeqAllToAll4D.apply(self.ulysses_group, output, 1, 2, False)
return self.o(output.flatten(2, 3))
@@ -373,6 +421,13 @@ class LingBotCrossAttention(nn.Module):
self.num_heads = num_heads
self.head_dim = dim // num_heads
self.num_local_heads = num_heads // tp_size
self.ulysses_world_size, self.ulysses_rank, self.ulysses_group = _ulysses_state()
if self.num_local_heads % self.ulysses_world_size:
raise ValueError(
"LingBot local cross-attention heads must be divisible by the Ulysses degree: "
f"heads={self.num_local_heads}, ulysses={self.ulysses_world_size}."
)
self.num_sp_heads = self.num_local_heads // self.ulysses_world_size
self.tp_inner_dim = self.num_local_heads * self.head_dim
self.q = ColumnParallelLinear(
@@ -410,9 +465,9 @@ class LingBotCrossAttention(nn.Module):
self.norm_q = _LingBotRMSNorm(self.tp_inner_dim, eps)
self.norm_k = _LingBotRMSNorm(self.tp_inner_dim, eps)
self.attn = Attention(
num_heads=self.num_local_heads,
num_heads=self.num_sp_heads,
head_size=self.head_dim,
num_kv_heads=self.num_local_heads,
num_kv_heads=self.num_sp_heads,
softmax_scale=self.head_dim**-0.5,
causal=False,
role="cross",
@@ -422,6 +477,14 @@ class LingBotCrossAttention(nn.Module):
disable_kv_quant=True,
)
def shard_kv_heads(self, value: torch.Tensor) -> torch.Tensor:
"""Select this Ulysses rank's projected K/V heads."""
if self.ulysses_world_size == 1:
return value
start = self.ulysses_rank * self.num_sp_heads
return value[:, :, start : start + self.num_sp_heads].clone(memory_format=torch.contiguous_format)
def forward(
self,
hidden_states: torch.Tensor,
@@ -431,6 +494,8 @@ class LingBotCrossAttention(nn.Module):
) -> tuple[torch.Tensor, LingBotAttentionCache]:
query = self.norm_q(self.q(hidden_states))
query = query.unflatten(2, (self.num_local_heads, self.head_dim))
if self.ulysses_world_size > 1:
query = SeqAllToAll4D.apply(self.ulysses_group, query, 2, 1, False)
# Text K/V is constant within a request and is projected once per layer.
if cache is None:
@@ -438,8 +503,8 @@ class LingBotCrossAttention(nn.Module):
raise ValueError("encoder_hidden_states are required when the cross-attention cache is empty.")
key = self.norm_k(self.k(encoder_hidden_states))
value = self.v(encoder_hidden_states)
key = key.unflatten(2, (self.num_local_heads, self.head_dim))
value = value.unflatten(2, (self.num_local_heads, self.head_dim))
key = self.shard_kv_heads(key.unflatten(2, (self.num_local_heads, self.head_dim)))
value = self.shard_kv_heads(value.unflatten(2, (self.num_local_heads, self.head_dim)))
cache = LingBotAttentionCache(
key=key,
value=value,
@@ -452,6 +517,8 @@ class LingBotCrossAttention(nn.Module):
value = cache.value[:, : cache.end]
output = self.attn(query, key, value)
if self.ulysses_world_size > 1:
output = SeqAllToAll4D.apply(self.ulysses_group, output, 1, 2, False)
return self.o(output.flatten(2, 3)), cache
@@ -540,7 +607,18 @@ class LingBotAttentionBlock(nn.Module):
rotary_emb: tuple[torch.Tensor, torch.Tensor],
) -> tuple[torch.Tensor, LingBotAttentionCache]:
batch_size, token_count, dim = hidden_states.shape
if (
timestep_projection.ndim != 4
or timestep_projection.shape[0] != batch_size
or timestep_projection.shape[2:] != (6, dim)
):
raise ValueError("LingBot timestep projection must have shape (batch, frames or tokens, 6, dim).")
num_frames = timestep_projection.shape[1]
if num_frames == 0 or token_count % num_frames:
raise ValueError("LingBot timestep projection must align with hidden tokens.")
if self.self_attn.ulysses_world_size > 1 and num_frames != token_count:
raise ValueError("LingBot Ulysses requires one timestep projection per local token.")
# Without SP, broadcast per frame. With SP, each entry represents one token.
tokens_per_frame = token_count // num_frames
# Timestep, camera, and text remain separate conditioning paths.
modulation = self.modulation.unsqueeze(1) + timestep_projection.float()
@@ -635,13 +713,26 @@ class _LingBotHead(nn.Module):
self,
hidden_states: torch.Tensor,
timestep_embedding: torch.Tensor,
*,
tokens_per_frame: int | None = None,
token_offset: int = 0,
) -> torch.Tensor:
num_frames = timestep_embedding.shape[1]
tokens_per_frame = hidden_states.shape[1] // num_frames
modulation = self.modulation.unsqueeze(1) + timestep_embedding.unsqueeze(2).float()
shift, scale = modulation.chunk(2, dim=2)
normalized = self.norm(hidden_states.float()).unflatten(1, (num_frames, tokens_per_frame))
normalized = (normalized * (1 + scale) + shift).flatten(1, 2).to(hidden_states.dtype)
normalized = self.norm(hidden_states.float())
if tokens_per_frame is None:
normalized = normalized.unflatten(1, (num_frames, -1))
normalized = (normalized * (1 + scale) + shift).flatten(1, 2)
else:
# A sequence shard can begin/end inside a frame.
frames = (
torch.arange(hidden_states.shape[1], device=hidden_states.device) + token_offset
) // tokens_per_frame
scale = scale.squeeze(2).index_select(1, frames)
shift = shift.squeeze(2).index_select(1, frames)
normalized = normalized * (1 + scale) + shift
normalized = normalized.to(hidden_states.dtype)
return self.head(normalized)
@@ -678,6 +769,16 @@ class CausalLingBotWorldTransformer3DModel(nn.Module):
_repeated_blocks = ["LingBotAttentionBlock"]
packed_modules_mapping = {"qkv": ["q", "k", "v"]}
_layerwise_offload_blocks_attrs = ["blocks"]
_sp_plan = {
"sp_prepare": {
0: SequenceParallelInput(split_dim=1, expected_dims=3, split_output=True),
1: SequenceParallelInput(split_dim=1, expected_dims=3, split_output=True),
2: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True),
3: SequenceParallelInput(split_dim=0, expected_dims=2, split_output=True),
4: SequenceParallelInput(split_dim=0, expected_dims=2, split_output=True),
},
"sp_output_gather": SequenceParallelOutput(gather_dim=1, expected_dims=3),
}
def __init__(
self,
@@ -768,6 +869,8 @@ class CausalLingBotWorldTransformer3DModel(nn.Module):
stride=patch_size,
)
self.patch_embedding_wancamctrl = _LingBotCameraPatchEmbedding(6 * 8 * 8, dim, patch_size)
self.sp_prepare = _LingBotSPPrepare()
self.sp_output_gather = nn.Identity()
self.c2ws_hidden_states_layer1 = ColumnParallelLinear(
dim,
dim,
@@ -904,8 +1007,7 @@ class CausalLingBotWorldTransformer3DModel(nn.Module):
self.config.local_attn_size if self.config.local_attn_size != -1 else self.config.sliding_window_num_frames
)
max_tokens = int(window_frames * post_patch_height * post_patch_width)
tp_size = get_tensor_model_parallel_world_size()
num_local_heads = self.config.num_attention_heads // tp_size
num_local_heads = self.blocks[0].self_attn.num_sp_heads
return allocate_lingbot_cache(
batch_size=batch_size,
num_layers=self.config.num_layers,
@@ -1138,6 +1240,33 @@ class CausalLingBotWorldTransformer3DModel(nn.Module):
query_len=hidden_states.shape[1],
)
cache.self_attention = [layer_context.to_layer_inputs() for layer_context in cache.self_attention]
sp_size = self.blocks[0].self_attn.ulysses_world_size
global_tokens = patched_frames * tokens_per_frame
hidden_dim = hidden_states.shape[-1]
rope_dim = rotary_emb[0].shape[-1]
if global_tokens % sp_size:
raise ValueError("LingBot Ulysses requires the token count to be divisible by its degree.")
hidden_states, camera_hidden_states, timestep_projection, cosine, sine = self.sp_prepare(
hidden_states,
camera_hidden_states,
timestep_projection,
rotary_emb,
)
if sp_size > 1:
local_tokens = global_tokens // sp_size
expected_hidden = (batch_size, local_tokens, hidden_dim)
if (
hidden_states.shape != expected_hidden
or camera_hidden_states.shape != expected_hidden
or timestep_projection.shape != (batch_size, local_tokens, 6, hidden_dim)
or cosine.shape != (local_tokens, rope_dim)
or sine.shape != (local_tokens, rope_dim)
):
raise RuntimeError(
"LingBot SP input hooks did not shard all conditioning tensors consistently; "
f"expected {local_tokens} tokens per rank. Check SP hook registration and input dimensions."
)
rotary_emb = (cosine, sine)
# Phase 3: each layer receives its own cache entry. Text K/V is passed
# only when absent; the returned cache is stored for subsequent DMD
# steps and causal blocks in this request.
@@ -1156,9 +1285,17 @@ class CausalLingBotWorldTransformer3DModel(nn.Module):
)
cache.cross_attention[index] = cross_cache
# Phase 4: map tokens to per-patch 16-channel flow values and restore
# [B, C, F, H, W] for the Pipeline's sampler update.
hidden_states = self.head(hidden_states, timestep_embedding)
# Phase 4: project local tokens before gathering the much narrower flow values.
hidden_states = self.head(
hidden_states,
timestep_embedding,
tokens_per_frame=tokens_per_frame if sp_size > 1 else None,
token_offset=get_sp_group().ulysses_rank * (global_tokens // sp_size) if sp_size > 1 else 0,
)
hidden_states = self.sp_output_gather(hidden_states)
projected_dim = self.config.out_channels * math.prod(self.config.patch_size)
if sp_size > 1 and hidden_states.shape != (batch_size, global_tokens, projected_dim):
raise RuntimeError("LingBot SP output hook did not gather the full projected token sequence.")
return self._unpatchify(
hidden_states,
batch_size=batch_size,