[Feature]: add Ulysses advanced_uaa mode (#1379)

Signed-off-by: dongbo910220 <1275604947@qq.com>
Signed-off-by: dongbo910220 <32610838+dongbo910220@users.noreply.github.com>
Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
dongbo910220andDidan Deng authored and GitHub committed 2026-03-24 14:55:54 +08:00
1 parent 4cfa9bf843
commit 50bd1961bf
17 files changed
+1064 -13

No files matched your search

@@ -315,10 +315,38 @@ class SequenceParallelSplitHook(ModelHook):
logger.warning_once(f"Expected tensor with {sp_input.expected_dims} dims, got {x.dim()}. Skipping split.")
return x
def _raise_strict_divisibility_error(*, dim: int, seq_len: int, sp_size: int) -> None:
# Keep message actionable: strict mode must be evenly shardable at the split hook level.
msg = (
"Sequence Parallel strict mode requires the sequence length to be evenly shardable at split time, "
f"but got seq_len={seq_len} (dim={dim}) not divisible by sequence_parallel_size={sp_size}. "
)
suggestions: list[str] = []
if self.config.ulysses_degree > 1:
suggestions.append("set parallel_config.ulysses_mode='advanced_uaa' (UAA)")
suggestions.append(
"enable auto_pad=True in the model _sp_plan (requires attention_mask support and ring_degree=1)"
)
suggestions.append("choose seq_len divisible by sequence_parallel_size (or adjust SP degrees)")
msg += "Try: " + "; ".join(suggestions) + "."
raise ValueError(msg)
def _maybe_validate_strict_divisibility(*, dim: int, seq_len: int) -> None:
from vllm_omni.diffusion.forward_context import get_ulysses_mode
if get_ulysses_mode(default="strict") != "strict":
return
sp_size = int(self.config.sequence_parallel_size)
if sp_size <= 1:
return
if seq_len < sp_size or (seq_len % sp_size) != 0:
_raise_strict_divisibility_error(dim=dim, seq_len=int(seq_len), sp_size=sp_size)
if isinstance(sp_input, SequenceParallelInput):
# Full split with optional auto-padding
if sp_input.auto_pad:
return self._shard_with_auto_pad(x, sp_input.split_dim)
_maybe_validate_strict_divisibility(dim=sp_input.split_dim, seq_len=x.size(sp_input.split_dim))
return sp_shard(x, sp_input.split_dim, validate=False)
elif isinstance(sp_input, SequenceParallelPartialInput):
# Partial split: keep text portion, split image portion
@@ -330,6 +358,7 @@ class SequenceParallelSplitHook(ModelHook):
image_part = x.narrow(dim, text_len, x.size(dim) - text_len)
# Only shard the image portion
_maybe_validate_strict_divisibility(dim=dim, seq_len=image_part.size(dim))
image_part_sharded = sp_shard(image_part, dim, validate=False)
# Concatenate back: [text_full, image_sharded]