[Feature]: add Ulysses advanced_uaa mode (#1379)

Signed-off-by: dongbo910220 <1275604947@qq.com>
Signed-off-by: dongbo910220 <32610838+dongbo910220@users.noreply.github.com>
Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
dongbo910220andDidan Deng authored and GitHub committed 2026-03-24 14:55:54 +08:00
1 parent 4cfa9bf843
commit 50bd1961bf
17 files changed
+1064 -13

No files matched your search

@@ -0,0 +1,308 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import os
import socket
import tempfile
import numpy as np
import pytest
import torch
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.parallel_state import (
destroy_distributed_env,
init_distributed_environment,
initialize_model_parallel,
)
from vllm_omni.diffusion.forward_context import get_forward_context, set_forward_context
from vllm_omni.platforms import current_omni_platform
def _find_free_port() -> int:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("127.0.0.1", 0))
return int(s.getsockname()[1])
def _set_dist_env(*, rank: int, world_size: int, master_port: int) -> None:
os.environ["RANK"] = str(rank)
os.environ["LOCAL_RANK"] = str(rank)
os.environ["WORLD_SIZE"] = str(world_size)
os.environ["MASTER_ADDR"] = "127.0.0.1"
os.environ["MASTER_PORT"] = str(master_port)
def _run_attention_case(
local_rank: int,
world_size: int,
master_port: int,
input_file: str,
output_file: str,
num_heads: int,
head_size: int,
ulysses_degree: int,
ulysses_mode: str,
ring_degree: int = 1,
split_sizes: list[int] | None = None,
sdp_kernel_mode: str = "math",
) -> None:
device = torch.device(f"{current_omni_platform.device_type}:{local_rank}")
current_omni_platform.set_device(device)
_set_dist_env(rank=local_rank, world_size=world_size, master_port=master_port)
os.environ["DIFFUSION_ATTENTION_BACKEND"] = "TORCH_SDPA"
init_distributed_environment(world_size=world_size, rank=local_rank)
initialize_model_parallel(
data_parallel_size=1,
cfg_parallel_size=1,
sequence_parallel_size=world_size,
ulysses_degree=ulysses_degree,
ring_degree=ring_degree,
tensor_parallel_size=1,
pipeline_parallel_size=1,
)
parallel_config = DiffusionParallelConfig(
pipeline_parallel_size=1,
data_parallel_size=1,
tensor_parallel_size=1,
sequence_parallel_size=world_size,
ulysses_degree=ulysses_degree,
ring_degree=ring_degree,
cfg_parallel_size=1,
ulysses_mode=ulysses_mode,
)
od_config = OmniDiffusionConfig(model="test", dtype=torch.float32, parallel_config=parallel_config)
with set_forward_context(omni_diffusion_config=od_config):
attn = Attention(
num_heads=num_heads,
head_size=head_size,
causal=False,
softmax_scale=1.0 / (head_size**0.5),
).to(device=device, dtype=torch.float32)
with np.load(input_file, allow_pickle=False) as payload:
q_full = torch.from_numpy(payload["q"]).to(device=device)
k_full = torch.from_numpy(payload["k"]).to(device=device)
v_full = torch.from_numpy(payload["v"]).to(device=device)
if world_size == 1:
q, k, v = q_full, k_full, v_full
else:
if split_sizes is None:
# NOTE: torch.chunk may return fewer than `world_size` chunks for some
# uneven lengths (e.g. seq_len=9, world_size=4 -> 3 chunks of len 3).
# We need an exact `world_size`-way split to simulate uneven SP shards.
q = torch.tensor_split(q_full, world_size, dim=1)[local_rank].contiguous()
k = torch.tensor_split(k_full, world_size, dim=1)[local_rank].contiguous()
v = torch.tensor_split(v_full, world_size, dim=1)[local_rank].contiguous()
else:
if len(split_sizes) != world_size:
raise ValueError(f"split_sizes length ({len(split_sizes)}) must equal world_size ({world_size}).")
if sum(int(x) for x in split_sizes) != q_full.shape[1]:
raise ValueError(
"split_sizes must sum to full seq_len "
f"(got sum={sum(int(x) for x in split_sizes)}, seq_len={q_full.shape[1]})."
)
q = torch.split(q_full, split_sizes, dim=1)[local_rank].contiguous()
k = torch.split(k_full, split_sizes, dim=1)[local_rank].contiguous()
v = torch.split(v_full, split_sizes, dim=1)[local_rank].contiguous()
# The Attention layer only enables SP communication when ForwardContext.sp_active is True.
# In production this is managed by SequenceParallelSplitHook/GatherHook, but here we
# shard manually for testing.
get_forward_context()._sp_shard_depth = 1
if sdp_kernel_mode == "math":
sdp_ctx = torch.backends.cuda.sdp_kernel(enable_flash=False, enable_mem_efficient=False, enable_math=True)
elif sdp_kernel_mode == "mem_efficient":
sdp_ctx = torch.backends.cuda.sdp_kernel(enable_flash=False, enable_mem_efficient=True, enable_math=False)
else:
raise ValueError(f"Invalid sdp_kernel_mode: {sdp_kernel_mode!r}")
with sdp_ctx:
out_local = attn(q, k, v).contiguous()
if world_size == 1:
out_full = out_local
else:
local_len = torch.tensor([out_local.shape[1]], device=device, dtype=torch.int64)
gathered_lens = [torch.empty_like(local_len) for _ in range(world_size)]
torch.distributed.all_gather(gathered_lens, local_len)
lens = [int(t.item()) for t in gathered_lens]
max_len = max(lens)
if out_local.shape[1] < max_len:
pad = max_len - out_local.shape[1]
out_local = torch.nn.functional.pad(out_local, (0, 0, 0, 0, 0, pad)).contiguous()
gathered = [torch.empty_like(out_local) for _ in range(world_size)]
torch.distributed.all_gather(gathered, out_local)
if local_rank == 0:
out_full = torch.cat([t[:, : lens[i]].contiguous() for i, t in enumerate(gathered)], dim=1)
else:
out_full = None
if local_rank == 0:
np.save(output_file, out_full.detach().cpu().numpy())
destroy_distributed_env()
@pytest.mark.parametrize(
"sp_world_size,seq_len,num_heads",
[
(2, 6, 3), # head_cnt not divisible by P=2
(2, 5, 4), # seq_len not divisible by P=2
(4, 9, 30), # Z-Image-like: head_cnt not divisible by P=4
(4, 10, 8), # seq_len not divisible by P=4
],
)
def test_ulysses_uaa_matches_baseline(sp_world_size: int, seq_len: int, num_heads: int) -> None:
if current_omni_platform.get_device_count() < sp_world_size:
pytest.skip(f"Test requires {sp_world_size} GPUs")
batch_size = 2
head_size = 8
base_port = _find_free_port()
sp_port = _find_free_port()
while sp_port == base_port:
sp_port = _find_free_port()
with tempfile.NamedTemporaryFile(delete=False, suffix=".npz") as f_in:
input_file = f_in.name
with tempfile.NamedTemporaryFile(delete=False, suffix=".npy") as f_base:
baseline_file = f_base.name
with tempfile.NamedTemporaryFile(delete=False, suffix=".npy") as f_sp:
sp_file = f_sp.name
try:
torch.manual_seed(0)
q = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
k = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
v = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
np.savez(input_file, q=q.numpy(), k=k.numpy(), v=v.numpy())
# Baseline (no SP)
torch.multiprocessing.spawn(
_run_attention_case,
args=(1, base_port, input_file, baseline_file, num_heads, head_size, 1, "strict"),
nprocs=1,
)
# SP (Ulysses-P with UAA)
torch.multiprocessing.spawn(
_run_attention_case,
args=(
sp_world_size,
sp_port,
input_file,
sp_file,
num_heads,
head_size,
sp_world_size,
"advanced_uaa",
),
nprocs=sp_world_size,
)
baseline = np.load(baseline_file, allow_pickle=False)
sp = np.load(sp_file, allow_pickle=False)
baseline_t = torch.from_numpy(baseline)
sp_t = torch.from_numpy(sp)
assert baseline_t.shape == sp_t.shape
torch.testing.assert_close(sp_t, baseline_t, atol=1e-5, rtol=1e-5)
finally:
for path in (input_file, baseline_file, sp_file):
try:
os.remove(path)
except OSError:
pass
def test_ulysses_uaa_hybrid_ring_matches_baseline() -> None:
sp_world_size = 4
ulysses_degree = 2
ring_degree = 2
if current_omni_platform.get_device_count() < sp_world_size:
pytest.skip(f"Test requires {sp_world_size} GPUs")
batch_size = 2
head_size = 8
seq_len = 10
num_heads = 3 # head_cnt not divisible by ulysses_degree=2 -> triggers head padding
# Ensure ring ranks see equal post-Ulysses seq_len:
# rank0/1 -> 3+2=5, rank2/3 -> 3+2=5
split_sizes = [3, 2, 3, 2]
base_port = _find_free_port()
sp_port = _find_free_port()
while sp_port == base_port:
sp_port = _find_free_port()
with tempfile.NamedTemporaryFile(delete=False, suffix=".npz") as f_in:
input_file = f_in.name
with tempfile.NamedTemporaryFile(delete=False, suffix=".npy") as f_base:
baseline_file = f_base.name
with tempfile.NamedTemporaryFile(delete=False, suffix=".npy") as f_sp:
sp_file = f_sp.name
try:
torch.manual_seed(0)
q = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
k = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
v = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
np.savez(input_file, q=q.numpy(), k=k.numpy(), v=v.numpy())
# Baseline (no SP)
torch.multiprocessing.spawn(
_run_attention_case,
args=(1, base_port, input_file, baseline_file, num_heads, head_size, 1, "strict", 1, None, "mem_efficient"),
nprocs=1,
)
# Hybrid SP: Ulysses (P=2) + Ring (P=2) with advanced_uaa
torch.multiprocessing.spawn(
_run_attention_case,
args=(
sp_world_size,
sp_port,
input_file,
sp_file,
num_heads,
head_size,
ulysses_degree,
"advanced_uaa",
ring_degree,
split_sizes,
"mem_efficient",
),
nprocs=sp_world_size,
)
baseline = np.load(baseline_file, allow_pickle=False)
sp = np.load(sp_file, allow_pickle=False)
baseline_t = torch.from_numpy(baseline)
sp_t = torch.from_numpy(sp)
assert baseline_t.shape == sp_t.shape
# Hybrid (Ulysses+Ring) typically has slightly larger numerical differences
# than pure Ulysses due to different communication/reduction order and
# the SDPA kernel path used by Ring attention. Use a looser tolerance to
# keep the test stable across GPUs/kernels while still catching regressions.
torch.testing.assert_close(sp_t, baseline_t, atol=5e-4, rtol=5e-4)
finally:
for path in (input_file, baseline_file, sp_file):
try:
os.remove(path)
except OSError:
pass
@@ -988,6 +988,46 @@ class TestDimensionValidation:
assert result.shape == tensor_4d.shape
@pytest.mark.cpu
class TestStrictModeSplitValidation:
"""Test strict mode divisibility validation in SequenceParallelSplitHook."""
def test_strict_mode_raises_on_non_divisible_seq_len(self):
"""Strict Ulysses-SP should fail fast when seq_len is not divisible by sp_size."""
from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelConfig
from vllm_omni.diffusion.forward_context import set_forward_context
from vllm_omni.diffusion.hooks.sequence_parallel import SequenceParallelSplitHook
class DummyModule(nn.Module):
def forward(self, x):
return x
metadata = {"x": SequenceParallelInput(split_dim=1, expected_dims=3)}
config = SequenceParallelConfig(ulysses_degree=2, ring_degree=1)
hook = SequenceParallelSplitHook(metadata, config)
hook.initialize_hook(DummyModule())
# seq_len=5 is not divisible by sp_size=2
x = torch.randn(1, 5, 8)
parallel_config = DiffusionParallelConfig(
pipeline_parallel_size=1,
data_parallel_size=1,
tensor_parallel_size=1,
sequence_parallel_size=2,
ulysses_degree=2,
ring_degree=1,
cfg_parallel_size=1,
ulysses_mode="strict",
)
od_config = OmniDiffusionConfig(model="test", dtype=torch.float32, parallel_config=parallel_config)
with set_forward_context(omni_diffusion_config=od_config):
with pytest.raises(ValueError, match=r"strict mode.*sequence_parallel_size"):
hook._prepare_sp_input(x, metadata["x"], (), {})
@pytest.mark.cpu
class TestSequenceParallelConfig:
"""Test SequenceParallelConfig dataclass."""
@@ -0,0 +1,232 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Perf smoke tests for Ulysses advanced_uaa communication overhead.
This test is intended for CI monitoring only:
- Print per-iteration timings and ratio vs strict Ulysses all-to-all.
- Use a loose sanity bound to catch gross regressions without flakiness.
"""
from __future__ import annotations
import os
import socket
from dataclasses import dataclass
import pytest
import torch
import torch.distributed as dist
from vllm_omni.diffusion.attention.parallel.ulysses import (
_all_gather_int,
_ulysses_all_to_all_any_o,
_ulysses_all_to_all_any_qkv,
)
from vllm_omni.diffusion.distributed.comm import SeqAllToAll4D
from vllm_omni.diffusion.distributed.parallel_state import (
destroy_distributed_env,
get_sp_group,
init_distributed_environment,
initialize_model_parallel,
)
from vllm_omni.platforms import current_omni_platform
def _find_free_port() -> int:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("127.0.0.1", 0))
return int(s.getsockname()[1])
def _set_dist_env(*, rank: int, world_size: int, master_port: int) -> None:
os.environ["RANK"] = str(rank)
os.environ["LOCAL_RANK"] = str(rank)
os.environ["WORLD_SIZE"] = str(world_size)
os.environ["MASTER_ADDR"] = "127.0.0.1"
os.environ["MASTER_PORT"] = str(master_port)
def _max_all_reduce(pg: dist.ProcessGroup, value: float, *, device: torch.device) -> float:
t = torch.tensor([value], device=device, dtype=torch.float32)
dist.all_reduce(t, op=dist.ReduceOp.MAX, group=pg)
return float(t.item())
@dataclass(frozen=True, slots=True)
class _PerfCase:
ulysses_degree: int
ring_degree: int
@property
def world_size(self) -> int:
return int(self.ulysses_degree * self.ring_degree)
PERF_CASES: list[_PerfCase] = [
_PerfCase(ulysses_degree=4, ring_degree=1),
_PerfCase(ulysses_degree=2, ring_degree=2),
]
@pytest.mark.parametrize("case", PERF_CASES)
def test_ulysses_advanced_uaa_comm_overhead(case: _PerfCase) -> None:
available_gpus = current_omni_platform.get_device_count()
if available_gpus < case.world_size:
pytest.skip(f"Requires {case.world_size} GPUs, got {available_gpus}")
master_port = _find_free_port()
torch.multiprocessing.spawn(
_perf_worker,
args=(case.world_size, master_port, case.ulysses_degree, case.ring_degree),
nprocs=case.world_size,
)
def _perf_worker(local_rank: int, world_size: int, master_port: int, ulysses_degree: int, ring_degree: int) -> None:
device = torch.device(f"{current_omni_platform.device_type}:{local_rank}")
current_omni_platform.set_device(device)
_set_dist_env(rank=local_rank, world_size=world_size, master_port=master_port)
try:
init_distributed_environment(world_size=world_size, rank=local_rank)
initialize_model_parallel(
data_parallel_size=1,
cfg_parallel_size=1,
sequence_parallel_size=world_size,
ulysses_degree=ulysses_degree,
ring_degree=ring_degree,
tensor_parallel_size=1,
pipeline_parallel_size=1,
)
sp_group = get_sp_group()
ulysses_pg = sp_group.ulysses_group
ring_pg = sp_group.ring_group
ulysses_world_size = dist.get_world_size(ulysses_pg)
ring_world_size = dist.get_world_size(ring_pg)
# A moderate tensor size to reduce timing noise while staying fast in CI.
bsz = 1
s_local = 256
head_cnt = 32 # divisible by ulysses_degree in both cases above
head_dim = 128
dtype = torch.float16
use_sync = False
torch.manual_seed(1234 + local_rank)
q = torch.randn(bsz, s_local, head_cnt, head_dim, device=device, dtype=dtype)
k = torch.randn_like(q)
v = torch.randn_like(q)
warmup_iters = 10
iters = 200
def strict_comm_step(q_in: torch.Tensor, k_in: torch.Tensor, v_in: torch.Tensor) -> torch.Tensor:
q_out = SeqAllToAll4D.apply(ulysses_pg, q_in, 2, 1, use_sync)
SeqAllToAll4D.apply(ulysses_pg, k_in, 2, 1, use_sync)
SeqAllToAll4D.apply(ulysses_pg, v_in, 2, 1, use_sync)
o_in = q_out # dummy
return SeqAllToAll4D.apply(ulysses_pg, o_in, 1, 2, use_sync)
def uaa_comm_step(q_in: torch.Tensor, k_in: torch.Tensor, v_in: torch.Tensor) -> torch.Tensor:
seq_lens = _all_gather_int(ulysses_pg, int(q_in.shape[1]), device=q_in.device)
s_global = int(sum(seq_lens))
if ring_world_size > 1:
ring_s_globals = _all_gather_int(ring_pg, s_global, device=q_in.device)
if len(set(ring_s_globals)) != 1:
raise RuntimeError(f"Unexpected hybrid ring post-Ulysses seq_len mismatch: {ring_s_globals}.")
q_out, orig_head_cnt = _ulysses_all_to_all_any_qkv(ulysses_pg, q_in, seq_lens=seq_lens, use_sync=use_sync)
_ulysses_all_to_all_any_qkv(ulysses_pg, k_in, seq_lens=seq_lens, use_sync=use_sync)
_ulysses_all_to_all_any_qkv(ulysses_pg, v_in, seq_lens=seq_lens, use_sync=use_sync)
o_in = q_out # dummy
return _ulysses_all_to_all_any_o(
ulysses_pg,
o_in,
seq_lens=seq_lens,
local_seq_len=int(s_local),
orig_head_cnt=int(orig_head_cnt),
use_sync=use_sync,
)
with torch.no_grad():
# Warmup (strict)
for _ in range(warmup_iters):
_ = strict_comm_step(q, k, v)
current_omni_platform.synchronize()
# Timed (strict)
t0 = torch.cuda.Event(enable_timing=True)
t1 = torch.cuda.Event(enable_timing=True)
t0.record()
for _ in range(iters):
_ = strict_comm_step(q, k, v)
t1.record()
current_omni_platform.synchronize()
strict_ms = float(t0.elapsed_time(t1)) / float(iters)
# Warmup (UAA)
for _ in range(warmup_iters):
_ = uaa_comm_step(q, k, v)
current_omni_platform.synchronize()
# Timed (UAA)
u0 = torch.cuda.Event(enable_timing=True)
u1 = torch.cuda.Event(enable_timing=True)
u0.record()
for _ in range(iters):
_ = uaa_comm_step(q, k, v)
u1.record()
current_omni_platform.synchronize()
uaa_ms = float(u0.elapsed_time(u1)) / float(iters)
# Reduce across ranks (use worst-rank to be conservative).
strict_ms_max = _max_all_reduce(dist.group.WORLD, strict_ms, device=device)
uaa_ms_max = _max_all_reduce(dist.group.WORLD, uaa_ms, device=device)
ratio = (uaa_ms_max / strict_ms_max) if strict_ms_max > 0 else float("inf")
# Approx bytes moved per iteration per-rank (send+recv) for 4x all-to-all.
elem_size = torch.tensor([], dtype=dtype).element_size()
per_a2a_bytes = int(q.numel()) * elem_size * 2
comm_bytes = int(4 * per_a2a_bytes)
strict_gbps = (comm_bytes / (strict_ms_max / 1000.0)) / 1e9 if strict_ms_max > 0 else 0.0
uaa_gbps = (comm_bytes / (uaa_ms_max / 1000.0)) / 1e9 if uaa_ms_max > 0 else 0.0
if dist.get_rank() == 0:
payload = {
"name": "ulysses_uaa_comm_perf",
"world_size": int(world_size),
"ulysses_degree": int(ulysses_degree),
"ring_degree": int(ring_degree),
"ulysses_world_size": int(ulysses_world_size),
"ring_world_size": int(ring_world_size),
"shape": {
"bsz": int(bsz),
"s_local": int(s_local),
"head_cnt": int(head_cnt),
"head_dim": int(head_dim),
},
"dtype": str(dtype),
"iters": int(iters),
"strict_ms_per_iter_max": float(strict_ms_max),
"uaa_ms_per_iter_max": float(uaa_ms_max),
"uaa_over_strict_ratio": float(ratio),
"comm_bytes_per_iter_per_rank": int(comm_bytes),
"strict_effective_gbps_per_rank": float(strict_gbps),
"uaa_effective_gbps_per_rank": float(uaa_gbps),
}
print(f"UAA_COMM_PERF_JSON={payload}")
# Loose bound: we only want to catch severe regressions. The printed JSON
# payload is used for monitoring smaller changes, while this threshold is
# intentionally generous to avoid flakiness across GPU types/drivers.
max_uaa_ms_per_iter = 10.0
assert uaa_ms_max < max_uaa_ms_per_iter, (
f"UAA comm too slow: uaa={uaa_ms_max:.3f}ms/iter, strict={strict_ms_max:.3f}ms/iter "
f"(cap={max_uaa_ms_per_iter:.1f}ms/iter)."
)
assert ratio < 3.0, f"UAA comm overhead too high: ratio={ratio:.3f}x (strict={strict_ms_max:.3f}ms)."
finally:
destroy_distributed_env()