[Perf][Diffusion] Skip the Ulysses length all-gather when shards are padded equal (#5717)
Signed-off-by: Xenoryn <timberh2000@outlook.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
e8526cc552
commit
039808e0d9
5 files changed
+537
-10
No files matched your search
@@ -14,7 +14,11 @@ from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.parallel.base import ParallelAttentionContext
|
||||
from vllm_omni.diffusion.distributed.comm import SeqAllToAll4D
|
||||
from vllm_omni.diffusion.distributed.group_coordinator import SequenceParallelGroupCoordinator
|
||||
from vllm_omni.diffusion.forward_context import get_ulysses_mode
|
||||
from vllm_omni.diffusion.forward_context import (
|
||||
get_forward_context,
|
||||
get_ulysses_mode,
|
||||
is_forward_context_available,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -142,7 +146,7 @@ def _ulysses_all_to_all_any_qkv(
|
||||
|
||||
input_split_sizes = [s_local] * world_size
|
||||
output_split_sizes = seq_lens
|
||||
s_global = int(sum(output_split_sizes))
|
||||
s_global = sum(output_split_sizes)
|
||||
|
||||
out = torch.empty((s_global, bsz, head_cnt_local, head_dim), device=x.device, dtype=x.dtype)
|
||||
dist.all_to_all_single(
|
||||
@@ -177,7 +181,7 @@ def _ulysses_all_to_all_any_o(
|
||||
return x
|
||||
|
||||
bsz, s_global, head_cnt_local, head_dim = x.shape
|
||||
s_local = int(local_seq_len)
|
||||
s_local = local_seq_len
|
||||
|
||||
# (B, S_global, H_local, D) -> (S_global, B, H_local, D)
|
||||
x_t = x.permute(1, 0, 2, 3).contiguous()
|
||||
@@ -411,9 +415,22 @@ class UlyssesParallelAttention:
|
||||
f"(got scatter_idx={self._scatter_idx}, gather_idx={self._gather_idx})."
|
||||
)
|
||||
|
||||
local_seq_len = int(query.shape[1])
|
||||
seq_lens = _all_gather_int(self._ulysses_pg, local_seq_len, device=query.device)
|
||||
s_global = int(sum(seq_lens))
|
||||
if is_forward_context_available() and get_forward_context().sp_rank_local_seq_lens_equal:
|
||||
# auto_pad already made every rank's shard the same length, so
|
||||
# the lengths are known without a collective. Keep local_seq_len
|
||||
# as a SymInt (no int()) so torch.compile(dynamic=True) does not
|
||||
# specialize on it, and skip the host sync + graph break that
|
||||
# _all_gather_int would introduce in every attention call.
|
||||
local_seq_len = query.shape[1]
|
||||
seq_lens = [local_seq_len] * ulysses_world_size
|
||||
else:
|
||||
local_seq_len = int(query.shape[1])
|
||||
seq_lens = _all_gather_int(
|
||||
self._ulysses_pg,
|
||||
local_seq_len,
|
||||
device=query.device,
|
||||
)
|
||||
s_global = sum(seq_lens)
|
||||
|
||||
# In hybrid Ulysses+Ring, Ring attention uses P2P send/recv with fixed-shape
|
||||
# buffers. This requires all ring ranks to have the same seq_len after the
|
||||
@@ -571,8 +588,8 @@ class UlyssesParallelAttention:
|
||||
joint_len=joint_len,
|
||||
joint_strategy=joint_strategy,
|
||||
use_uaa=(mode == "advanced_uaa"),
|
||||
uaa_seq_lens=tuple(int(x) for x in seq_lens) if mode == "advanced_uaa" else (),
|
||||
uaa_local_seq_len=int(local_seq_len) if mode == "advanced_uaa" else 0,
|
||||
uaa_seq_lens=tuple(seq_lens) if mode == "advanced_uaa" else (),
|
||||
uaa_local_seq_len=local_seq_len if mode == "advanced_uaa" else 0,
|
||||
orig_head_cnt=int(orig_head_cnt) if mode == "advanced_uaa" else 0,
|
||||
joint_orig_head_cnt=int(joint_orig_head_cnt) if mode == "advanced_uaa" else 0,
|
||||
)
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
@@ -66,6 +66,10 @@ class ForwardContext:
|
||||
# Tracks the depth of SP sharding - incremented on shard, decremented on gather
|
||||
# Used by attention layers to determine if SP communication should be enabled
|
||||
_sp_shard_depth: int = 0
|
||||
# One entry per active SP split boundary: whether it auto-pads, which makes
|
||||
# every rank's shard the same length and lets attention collectives skip a
|
||||
# runtime length all-gather. Pushed on split, popped on gather.
|
||||
_sp_equal_pad_stack: list[bool] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def sp_active(self) -> bool:
|
||||
@@ -88,6 +92,15 @@ class ForwardContext:
|
||||
sp_size = self.omni_diffusion_config.parallel_config.sequence_parallel_size
|
||||
return sp_size is not None and sp_size > 1
|
||||
|
||||
@property
|
||||
def sp_rank_local_seq_lens_equal(self) -> bool:
|
||||
"""Whether every active SP boundary guarantees equal local shard sizes.
|
||||
|
||||
Only framework-managed auto_pad boundaries provide this contract; a
|
||||
region that also shards manually keeps the dynamic length exchange.
|
||||
"""
|
||||
return bool(self._sp_equal_pad_stack) and all(self._sp_equal_pad_stack)
|
||||
|
||||
def __post_init__(self):
|
||||
pass
|
||||
|
||||
|
||||
@@ -264,6 +264,7 @@ class SequenceParallelSplitHook(ModelHook):
|
||||
|
||||
output_list = [output] if is_tensor else list(output)
|
||||
actually_sharded = False
|
||||
equal_rank_seq_lens = True
|
||||
|
||||
for index, spm in self.metadata.items():
|
||||
if not isinstance(index, int):
|
||||
@@ -277,10 +278,13 @@ class SequenceParallelSplitHook(ModelHook):
|
||||
output_list[index] = self._prepare_sp_input(original, spm, self._last_args, self._last_kwargs)
|
||||
if output_list[index] is not original:
|
||||
actually_sharded = True
|
||||
equal_rank_seq_lens &= isinstance(spm, SequenceParallelInput) and spm.auto_pad
|
||||
|
||||
# Mark SP as active only if at least one tensor was actually sharded
|
||||
if actually_sharded and is_forward_context_available():
|
||||
get_forward_context()._sp_shard_depth += 1
|
||||
ctx = get_forward_context()
|
||||
ctx._sp_shard_depth += 1
|
||||
ctx._sp_equal_pad_stack.append(equal_rank_seq_lens)
|
||||
|
||||
return output_list[0] if is_tensor else type(output)(output_list)
|
||||
|
||||
@@ -553,6 +557,8 @@ class SequenceParallelGatherHook(ModelHook):
|
||||
if actually_gathered and is_forward_context_available():
|
||||
ctx = get_forward_context()
|
||||
ctx._sp_shard_depth = max(0, ctx._sp_shard_depth - 1)
|
||||
if ctx._sp_equal_pad_stack:
|
||||
ctx._sp_equal_pad_stack.pop()
|
||||
|
||||
return output[0] if is_tensor else type(output)(output)
|
||||
|
||||
|
||||
Reference in new issue
Block a user