[Bugfix] Fix Helios USP: per-component split for correct sequence parallelism (#6930)

Signed-off-by: yancaocn <yancaochn@163.com>
Co-authored-by: yancaocn <yancaochn@163.com>
This commit is contained in:
Yan Caoandyancaocn authored and GitHub committed 2026-09-11 11:43:34 +08:00
1 parent 3d952d1334
commit bad50980c9
4 files changed
+561 -12

No files matched your search

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
# Adapted from Helios (https://github.com/BestWishYsh/Helios)
import math
@@ -27,10 +27,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.diffusion.distributed.sp_plan import (
SequenceParallelInput,
SequenceParallelOutput,
)
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelOutput
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import (
@@ -448,6 +445,12 @@ class HeliosCrossAttention(nn.Module):
num_kv_heads=self.num_heads,
softmax_scale=1.0 / (head_dim**0.5),
causal=False,
# Text K/V is replicated on every SP rank, so Ulysses must not be
# used here: its all-to-all would treat the per-rank replicas as
# sequence shards and ws-fold duplicate the keys. Sharded Q over
# full replicated K/V is correct locally and needs no
# communication (same as Wan2.2 cross-attn).
skip_sequence_parallel=True,
)
def project_kv(self, encoder_hidden_states: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
@@ -624,16 +627,50 @@ class HeliosTransformer3DModel(nn.Module):
_hsdp_shard_conditions = [_is_transformer_block]
_sp_plan = {
"rope": {
0: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True, auto_pad=True),
1: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True, auto_pad=True),
},
"blocks.0": {
"hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, auto_pad=True),
},
# "rope" and "blocks.0" are intentionally omitted.
# The rope hook splits along dim=1 of the 5D output [B, D, T, H, W],
# which is the frequency dimension, NOT the sequence dimension.
# The blocks.0 hook splits the concatenated hidden_states as a whole,
# which can put all history tokens in one rank (original_context_length=0).
# Instead, hidden_states and rotary_emb are split per-component in
# forward() after flatten+transpose, ensuring each rank gets half
# of each component (history and current).
"proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
}
def _sp_split_seq(self, x: torch.Tensor) -> torch.Tensor:
"""Split tensor along sequence dim (dim=1) for Ulysses SP.
If seq_len is not divisible by world_size, replicate the last
token(s) to pad to a divisible length. This avoids crashing
on non-divisible history components (which use different patch
sizes and thus produce different token counts).
Residual caveat: the replicated padding tokens attend unmasked
in attn1 on the last rank (at most ws-1 replicas per component).
The impact is bounded, but this is the first place to look if
odd resolutions ever show quality drift.
"""
from vllm_omni.diffusion.distributed.parallel_state import (
get_sequence_parallel_rank,
get_sequence_parallel_world_size,
)
ws = get_sequence_parallel_world_size()
if ws > 1 and x.dim() >= 2 and x.shape[1] > 0:
seq_len = x.shape[1]
remainder = seq_len % ws
if remainder != 0:
pad = ws - remainder
# Replicate the last token to fill the padding
last = x[:, -1:, ...].expand(-1, pad, *([-1] * (x.dim() - 2)))
x = torch.cat([x, last], dim=1)
seq_len = x.shape[1]
r = get_sequence_parallel_rank()
n = seq_len // ws
x = x[:, r * n : (r + 1) * n, ...].contiguous()
return x
def __init__(
self,
patch_size: tuple[int, ...] = (1, 2, 2),
@@ -860,6 +897,9 @@ class HeliosTransformer3DModel(nn.Module):
device=hidden_states.device,
)
rotary_emb = rotary_emb.flatten(2).transpose(1, 2)
# USP: per-component split (each rank gets half of current frames)
hidden_states = self._sp_split_seq(hidden_states)
rotary_emb = self._sp_split_seq(rotary_emb)
original_context_length = hidden_states.shape[1]
# 2. Process short history latents
@@ -876,6 +916,9 @@ class HeliosTransformer3DModel(nn.Module):
device=latents_history_short.device,
)
rotary_emb_history_short = rotary_emb_history_short.flatten(2).transpose(1, 2)
# USP: per-component split
latents_history_short = self._sp_split_seq(latents_history_short)
rotary_emb_history_short = self._sp_split_seq(rotary_emb_history_short)
hidden_states = torch.cat([latents_history_short, hidden_states], dim=1)
rotary_emb = torch.cat([rotary_emb_history_short, rotary_emb], dim=1)
@@ -896,6 +939,9 @@ class HeliosTransformer3DModel(nn.Module):
rotary_emb_history_mid = pad_for_3d_conv(rotary_emb_history_mid, (2, 2, 2))
rotary_emb_history_mid = center_down_sample_3d(rotary_emb_history_mid, (2, 2, 2))
rotary_emb_history_mid = rotary_emb_history_mid.flatten(2).transpose(1, 2)
# USP: per-component split
latents_history_mid = self._sp_split_seq(latents_history_mid)
rotary_emb_history_mid = self._sp_split_seq(rotary_emb_history_mid)
hidden_states = torch.cat([latents_history_mid, hidden_states], dim=1)
rotary_emb = torch.cat([rotary_emb_history_mid, rotary_emb], dim=1)
@@ -916,6 +962,9 @@ class HeliosTransformer3DModel(nn.Module):
rotary_emb_history_long = pad_for_3d_conv(rotary_emb_history_long, (4, 4, 4))
rotary_emb_history_long = center_down_sample_3d(rotary_emb_history_long, (4, 4, 4))
rotary_emb_history_long = rotary_emb_history_long.flatten(2).transpose(1, 2)
# USP: per-component split
latents_history_long = self._sp_split_seq(latents_history_long)
rotary_emb_history_long = self._sp_split_seq(rotary_emb_history_long)
hidden_states = torch.cat([latents_history_long, hidden_states], dim=1)
rotary_emb = torch.cat([rotary_emb_history_long, rotary_emb], dim=1)
@@ -958,6 +1007,18 @@ class HeliosTransformer3DModel(nn.Module):
timestep_proj = timestep_proj.permute(0, 2, 1, 3)
# 6. Transformer blocks
# Manually increment _sp_shard_depth so that attention layers
# know SP is active and perform Ulysses All-to-All communication.
# Normally this is done by split_output=True hooks (e.g. rope),
# but those are removed because they split the wrong dimension.
from vllm_omni.diffusion.forward_context import (
get_forward_context,
is_forward_context_available,
)
if is_forward_context_available():
get_forward_context()._sp_shard_depth += 1
hidden_states = hidden_states.contiguous()
encoder_hidden_states = encoder_hidden_states.contiguous()
rotary_emb = rotary_emb.contiguous()
@@ -979,6 +1040,11 @@ class HeliosTransformer3DModel(nn.Module):
hidden_states = self.proj_out(hidden_states)
# 8. Unpatchify
# proj_out gather may include padded tokens from non-divisible
# sequences. Slice to the expected size before reshape.
expected_seq = post_patch_num_frames * post_patch_height * post_patch_width
if hidden_states.shape[1] > expected_seq:
hidden_states = hidden_states[:, :expected_seq, :]
hidden_states = hidden_states.reshape(
batch_size, post_patch_num_frames, post_patch_height, post_patch_width, p_t, p_h, p_w, -1
)