[Feature]: add Ulysses advanced_uaa mode (#1379)
Signed-off-by: dongbo910220 <1275604947@qq.com> Signed-off-by: dongbo910220 <32610838+dongbo910220@users.noreply.github.com> Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
1 parent
4cfa9bf843
commit
50bd1961bf
17 files changed
+1064
-13
No files matched your search
@@ -0,0 +1,308 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import socket
|
||||
import tempfile
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
destroy_distributed_env,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm_omni.diffusion.forward_context import get_forward_context, set_forward_context
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
def _find_free_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
return int(s.getsockname()[1])
|
||||
|
||||
|
||||
def _set_dist_env(*, rank: int, world_size: int, master_port: int) -> None:
|
||||
os.environ["RANK"] = str(rank)
|
||||
os.environ["LOCAL_RANK"] = str(rank)
|
||||
os.environ["WORLD_SIZE"] = str(world_size)
|
||||
os.environ["MASTER_ADDR"] = "127.0.0.1"
|
||||
os.environ["MASTER_PORT"] = str(master_port)
|
||||
|
||||
|
||||
def _run_attention_case(
|
||||
local_rank: int,
|
||||
world_size: int,
|
||||
master_port: int,
|
||||
input_file: str,
|
||||
output_file: str,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
ulysses_degree: int,
|
||||
ulysses_mode: str,
|
||||
ring_degree: int = 1,
|
||||
split_sizes: list[int] | None = None,
|
||||
sdp_kernel_mode: str = "math",
|
||||
) -> None:
|
||||
device = torch.device(f"{current_omni_platform.device_type}:{local_rank}")
|
||||
current_omni_platform.set_device(device)
|
||||
|
||||
_set_dist_env(rank=local_rank, world_size=world_size, master_port=master_port)
|
||||
os.environ["DIFFUSION_ATTENTION_BACKEND"] = "TORCH_SDPA"
|
||||
|
||||
init_distributed_environment(world_size=world_size, rank=local_rank)
|
||||
initialize_model_parallel(
|
||||
data_parallel_size=1,
|
||||
cfg_parallel_size=1,
|
||||
sequence_parallel_size=world_size,
|
||||
ulysses_degree=ulysses_degree,
|
||||
ring_degree=ring_degree,
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
)
|
||||
|
||||
parallel_config = DiffusionParallelConfig(
|
||||
pipeline_parallel_size=1,
|
||||
data_parallel_size=1,
|
||||
tensor_parallel_size=1,
|
||||
sequence_parallel_size=world_size,
|
||||
ulysses_degree=ulysses_degree,
|
||||
ring_degree=ring_degree,
|
||||
cfg_parallel_size=1,
|
||||
ulysses_mode=ulysses_mode,
|
||||
)
|
||||
od_config = OmniDiffusionConfig(model="test", dtype=torch.float32, parallel_config=parallel_config)
|
||||
|
||||
with set_forward_context(omni_diffusion_config=od_config):
|
||||
attn = Attention(
|
||||
num_heads=num_heads,
|
||||
head_size=head_size,
|
||||
causal=False,
|
||||
softmax_scale=1.0 / (head_size**0.5),
|
||||
).to(device=device, dtype=torch.float32)
|
||||
|
||||
with np.load(input_file, allow_pickle=False) as payload:
|
||||
q_full = torch.from_numpy(payload["q"]).to(device=device)
|
||||
k_full = torch.from_numpy(payload["k"]).to(device=device)
|
||||
v_full = torch.from_numpy(payload["v"]).to(device=device)
|
||||
|
||||
if world_size == 1:
|
||||
q, k, v = q_full, k_full, v_full
|
||||
else:
|
||||
if split_sizes is None:
|
||||
# NOTE: torch.chunk may return fewer than `world_size` chunks for some
|
||||
# uneven lengths (e.g. seq_len=9, world_size=4 -> 3 chunks of len 3).
|
||||
# We need an exact `world_size`-way split to simulate uneven SP shards.
|
||||
q = torch.tensor_split(q_full, world_size, dim=1)[local_rank].contiguous()
|
||||
k = torch.tensor_split(k_full, world_size, dim=1)[local_rank].contiguous()
|
||||
v = torch.tensor_split(v_full, world_size, dim=1)[local_rank].contiguous()
|
||||
else:
|
||||
if len(split_sizes) != world_size:
|
||||
raise ValueError(f"split_sizes length ({len(split_sizes)}) must equal world_size ({world_size}).")
|
||||
if sum(int(x) for x in split_sizes) != q_full.shape[1]:
|
||||
raise ValueError(
|
||||
"split_sizes must sum to full seq_len "
|
||||
f"(got sum={sum(int(x) for x in split_sizes)}, seq_len={q_full.shape[1]})."
|
||||
)
|
||||
q = torch.split(q_full, split_sizes, dim=1)[local_rank].contiguous()
|
||||
k = torch.split(k_full, split_sizes, dim=1)[local_rank].contiguous()
|
||||
v = torch.split(v_full, split_sizes, dim=1)[local_rank].contiguous()
|
||||
# The Attention layer only enables SP communication when ForwardContext.sp_active is True.
|
||||
# In production this is managed by SequenceParallelSplitHook/GatherHook, but here we
|
||||
# shard manually for testing.
|
||||
get_forward_context()._sp_shard_depth = 1
|
||||
|
||||
if sdp_kernel_mode == "math":
|
||||
sdp_ctx = torch.backends.cuda.sdp_kernel(enable_flash=False, enable_mem_efficient=False, enable_math=True)
|
||||
elif sdp_kernel_mode == "mem_efficient":
|
||||
sdp_ctx = torch.backends.cuda.sdp_kernel(enable_flash=False, enable_mem_efficient=True, enable_math=False)
|
||||
else:
|
||||
raise ValueError(f"Invalid sdp_kernel_mode: {sdp_kernel_mode!r}")
|
||||
|
||||
with sdp_ctx:
|
||||
out_local = attn(q, k, v).contiguous()
|
||||
|
||||
if world_size == 1:
|
||||
out_full = out_local
|
||||
else:
|
||||
local_len = torch.tensor([out_local.shape[1]], device=device, dtype=torch.int64)
|
||||
gathered_lens = [torch.empty_like(local_len) for _ in range(world_size)]
|
||||
torch.distributed.all_gather(gathered_lens, local_len)
|
||||
lens = [int(t.item()) for t in gathered_lens]
|
||||
max_len = max(lens)
|
||||
|
||||
if out_local.shape[1] < max_len:
|
||||
pad = max_len - out_local.shape[1]
|
||||
out_local = torch.nn.functional.pad(out_local, (0, 0, 0, 0, 0, pad)).contiguous()
|
||||
|
||||
gathered = [torch.empty_like(out_local) for _ in range(world_size)]
|
||||
torch.distributed.all_gather(gathered, out_local)
|
||||
if local_rank == 0:
|
||||
out_full = torch.cat([t[:, : lens[i]].contiguous() for i, t in enumerate(gathered)], dim=1)
|
||||
else:
|
||||
out_full = None
|
||||
|
||||
if local_rank == 0:
|
||||
np.save(output_file, out_full.detach().cpu().numpy())
|
||||
|
||||
destroy_distributed_env()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sp_world_size,seq_len,num_heads",
|
||||
[
|
||||
(2, 6, 3), # head_cnt not divisible by P=2
|
||||
(2, 5, 4), # seq_len not divisible by P=2
|
||||
(4, 9, 30), # Z-Image-like: head_cnt not divisible by P=4
|
||||
(4, 10, 8), # seq_len not divisible by P=4
|
||||
],
|
||||
)
|
||||
def test_ulysses_uaa_matches_baseline(sp_world_size: int, seq_len: int, num_heads: int) -> None:
|
||||
if current_omni_platform.get_device_count() < sp_world_size:
|
||||
pytest.skip(f"Test requires {sp_world_size} GPUs")
|
||||
|
||||
batch_size = 2
|
||||
head_size = 8
|
||||
|
||||
base_port = _find_free_port()
|
||||
sp_port = _find_free_port()
|
||||
while sp_port == base_port:
|
||||
sp_port = _find_free_port()
|
||||
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".npz") as f_in:
|
||||
input_file = f_in.name
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".npy") as f_base:
|
||||
baseline_file = f_base.name
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".npy") as f_sp:
|
||||
sp_file = f_sp.name
|
||||
|
||||
try:
|
||||
torch.manual_seed(0)
|
||||
q = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
|
||||
k = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
|
||||
v = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
|
||||
np.savez(input_file, q=q.numpy(), k=k.numpy(), v=v.numpy())
|
||||
|
||||
# Baseline (no SP)
|
||||
torch.multiprocessing.spawn(
|
||||
_run_attention_case,
|
||||
args=(1, base_port, input_file, baseline_file, num_heads, head_size, 1, "strict"),
|
||||
nprocs=1,
|
||||
)
|
||||
|
||||
# SP (Ulysses-P with UAA)
|
||||
torch.multiprocessing.spawn(
|
||||
_run_attention_case,
|
||||
args=(
|
||||
sp_world_size,
|
||||
sp_port,
|
||||
input_file,
|
||||
sp_file,
|
||||
num_heads,
|
||||
head_size,
|
||||
sp_world_size,
|
||||
"advanced_uaa",
|
||||
),
|
||||
nprocs=sp_world_size,
|
||||
)
|
||||
|
||||
baseline = np.load(baseline_file, allow_pickle=False)
|
||||
sp = np.load(sp_file, allow_pickle=False)
|
||||
|
||||
baseline_t = torch.from_numpy(baseline)
|
||||
sp_t = torch.from_numpy(sp)
|
||||
assert baseline_t.shape == sp_t.shape
|
||||
torch.testing.assert_close(sp_t, baseline_t, atol=1e-5, rtol=1e-5)
|
||||
finally:
|
||||
for path in (input_file, baseline_file, sp_file):
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def test_ulysses_uaa_hybrid_ring_matches_baseline() -> None:
|
||||
sp_world_size = 4
|
||||
ulysses_degree = 2
|
||||
ring_degree = 2
|
||||
|
||||
if current_omni_platform.get_device_count() < sp_world_size:
|
||||
pytest.skip(f"Test requires {sp_world_size} GPUs")
|
||||
|
||||
batch_size = 2
|
||||
head_size = 8
|
||||
seq_len = 10
|
||||
num_heads = 3 # head_cnt not divisible by ulysses_degree=2 -> triggers head padding
|
||||
|
||||
# Ensure ring ranks see equal post-Ulysses seq_len:
|
||||
# rank0/1 -> 3+2=5, rank2/3 -> 3+2=5
|
||||
split_sizes = [3, 2, 3, 2]
|
||||
|
||||
base_port = _find_free_port()
|
||||
sp_port = _find_free_port()
|
||||
while sp_port == base_port:
|
||||
sp_port = _find_free_port()
|
||||
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".npz") as f_in:
|
||||
input_file = f_in.name
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".npy") as f_base:
|
||||
baseline_file = f_base.name
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".npy") as f_sp:
|
||||
sp_file = f_sp.name
|
||||
|
||||
try:
|
||||
torch.manual_seed(0)
|
||||
q = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
|
||||
k = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
|
||||
v = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
|
||||
np.savez(input_file, q=q.numpy(), k=k.numpy(), v=v.numpy())
|
||||
|
||||
# Baseline (no SP)
|
||||
torch.multiprocessing.spawn(
|
||||
_run_attention_case,
|
||||
args=(1, base_port, input_file, baseline_file, num_heads, head_size, 1, "strict", 1, None, "mem_efficient"),
|
||||
nprocs=1,
|
||||
)
|
||||
|
||||
# Hybrid SP: Ulysses (P=2) + Ring (P=2) with advanced_uaa
|
||||
torch.multiprocessing.spawn(
|
||||
_run_attention_case,
|
||||
args=(
|
||||
sp_world_size,
|
||||
sp_port,
|
||||
input_file,
|
||||
sp_file,
|
||||
num_heads,
|
||||
head_size,
|
||||
ulysses_degree,
|
||||
"advanced_uaa",
|
||||
ring_degree,
|
||||
split_sizes,
|
||||
"mem_efficient",
|
||||
),
|
||||
nprocs=sp_world_size,
|
||||
)
|
||||
|
||||
baseline = np.load(baseline_file, allow_pickle=False)
|
||||
sp = np.load(sp_file, allow_pickle=False)
|
||||
|
||||
baseline_t = torch.from_numpy(baseline)
|
||||
sp_t = torch.from_numpy(sp)
|
||||
assert baseline_t.shape == sp_t.shape
|
||||
# Hybrid (Ulysses+Ring) typically has slightly larger numerical differences
|
||||
# than pure Ulysses due to different communication/reduction order and
|
||||
# the SDPA kernel path used by Ring attention. Use a looser tolerance to
|
||||
# keep the test stable across GPUs/kernels while still catching regressions.
|
||||
torch.testing.assert_close(sp_t, baseline_t, atol=5e-4, rtol=5e-4)
|
||||
finally:
|
||||
for path in (input_file, baseline_file, sp_file):
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
@@ -988,6 +988,46 @@ class TestDimensionValidation:
|
||||
assert result.shape == tensor_4d.shape
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
class TestStrictModeSplitValidation:
|
||||
"""Test strict mode divisibility validation in SequenceParallelSplitHook."""
|
||||
|
||||
def test_strict_mode_raises_on_non_divisible_seq_len(self):
|
||||
"""Strict Ulysses-SP should fail fast when seq_len is not divisible by sp_size."""
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelConfig
|
||||
from vllm_omni.diffusion.forward_context import set_forward_context
|
||||
from vllm_omni.diffusion.hooks.sequence_parallel import SequenceParallelSplitHook
|
||||
|
||||
class DummyModule(nn.Module):
|
||||
def forward(self, x):
|
||||
return x
|
||||
|
||||
metadata = {"x": SequenceParallelInput(split_dim=1, expected_dims=3)}
|
||||
config = SequenceParallelConfig(ulysses_degree=2, ring_degree=1)
|
||||
hook = SequenceParallelSplitHook(metadata, config)
|
||||
hook.initialize_hook(DummyModule())
|
||||
|
||||
# seq_len=5 is not divisible by sp_size=2
|
||||
x = torch.randn(1, 5, 8)
|
||||
|
||||
parallel_config = DiffusionParallelConfig(
|
||||
pipeline_parallel_size=1,
|
||||
data_parallel_size=1,
|
||||
tensor_parallel_size=1,
|
||||
sequence_parallel_size=2,
|
||||
ulysses_degree=2,
|
||||
ring_degree=1,
|
||||
cfg_parallel_size=1,
|
||||
ulysses_mode="strict",
|
||||
)
|
||||
od_config = OmniDiffusionConfig(model="test", dtype=torch.float32, parallel_config=parallel_config)
|
||||
|
||||
with set_forward_context(omni_diffusion_config=od_config):
|
||||
with pytest.raises(ValueError, match=r"strict mode.*sequence_parallel_size"):
|
||||
hook._prepare_sp_input(x, metadata["x"], (), {})
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
class TestSequenceParallelConfig:
|
||||
"""Test SequenceParallelConfig dataclass."""
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Perf smoke tests for Ulysses advanced_uaa communication overhead.
|
||||
|
||||
This test is intended for CI monitoring only:
|
||||
- Print per-iteration timings and ratio vs strict Ulysses all-to-all.
|
||||
- Use a loose sanity bound to catch gross regressions without flakiness.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm_omni.diffusion.attention.parallel.ulysses import (
|
||||
_all_gather_int,
|
||||
_ulysses_all_to_all_any_o,
|
||||
_ulysses_all_to_all_any_qkv,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.comm import SeqAllToAll4D
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
destroy_distributed_env,
|
||||
get_sp_group,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
def _find_free_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
return int(s.getsockname()[1])
|
||||
|
||||
|
||||
def _set_dist_env(*, rank: int, world_size: int, master_port: int) -> None:
|
||||
os.environ["RANK"] = str(rank)
|
||||
os.environ["LOCAL_RANK"] = str(rank)
|
||||
os.environ["WORLD_SIZE"] = str(world_size)
|
||||
os.environ["MASTER_ADDR"] = "127.0.0.1"
|
||||
os.environ["MASTER_PORT"] = str(master_port)
|
||||
|
||||
|
||||
def _max_all_reduce(pg: dist.ProcessGroup, value: float, *, device: torch.device) -> float:
|
||||
t = torch.tensor([value], device=device, dtype=torch.float32)
|
||||
dist.all_reduce(t, op=dist.ReduceOp.MAX, group=pg)
|
||||
return float(t.item())
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PerfCase:
|
||||
ulysses_degree: int
|
||||
ring_degree: int
|
||||
|
||||
@property
|
||||
def world_size(self) -> int:
|
||||
return int(self.ulysses_degree * self.ring_degree)
|
||||
|
||||
|
||||
PERF_CASES: list[_PerfCase] = [
|
||||
_PerfCase(ulysses_degree=4, ring_degree=1),
|
||||
_PerfCase(ulysses_degree=2, ring_degree=2),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", PERF_CASES)
|
||||
def test_ulysses_advanced_uaa_comm_overhead(case: _PerfCase) -> None:
|
||||
available_gpus = current_omni_platform.get_device_count()
|
||||
if available_gpus < case.world_size:
|
||||
pytest.skip(f"Requires {case.world_size} GPUs, got {available_gpus}")
|
||||
|
||||
master_port = _find_free_port()
|
||||
torch.multiprocessing.spawn(
|
||||
_perf_worker,
|
||||
args=(case.world_size, master_port, case.ulysses_degree, case.ring_degree),
|
||||
nprocs=case.world_size,
|
||||
)
|
||||
|
||||
|
||||
def _perf_worker(local_rank: int, world_size: int, master_port: int, ulysses_degree: int, ring_degree: int) -> None:
|
||||
device = torch.device(f"{current_omni_platform.device_type}:{local_rank}")
|
||||
current_omni_platform.set_device(device)
|
||||
|
||||
_set_dist_env(rank=local_rank, world_size=world_size, master_port=master_port)
|
||||
|
||||
try:
|
||||
init_distributed_environment(world_size=world_size, rank=local_rank)
|
||||
initialize_model_parallel(
|
||||
data_parallel_size=1,
|
||||
cfg_parallel_size=1,
|
||||
sequence_parallel_size=world_size,
|
||||
ulysses_degree=ulysses_degree,
|
||||
ring_degree=ring_degree,
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
)
|
||||
|
||||
sp_group = get_sp_group()
|
||||
ulysses_pg = sp_group.ulysses_group
|
||||
ring_pg = sp_group.ring_group
|
||||
|
||||
ulysses_world_size = dist.get_world_size(ulysses_pg)
|
||||
ring_world_size = dist.get_world_size(ring_pg)
|
||||
|
||||
# A moderate tensor size to reduce timing noise while staying fast in CI.
|
||||
bsz = 1
|
||||
s_local = 256
|
||||
head_cnt = 32 # divisible by ulysses_degree in both cases above
|
||||
head_dim = 128
|
||||
dtype = torch.float16
|
||||
use_sync = False
|
||||
|
||||
torch.manual_seed(1234 + local_rank)
|
||||
q = torch.randn(bsz, s_local, head_cnt, head_dim, device=device, dtype=dtype)
|
||||
k = torch.randn_like(q)
|
||||
v = torch.randn_like(q)
|
||||
|
||||
warmup_iters = 10
|
||||
iters = 200
|
||||
|
||||
def strict_comm_step(q_in: torch.Tensor, k_in: torch.Tensor, v_in: torch.Tensor) -> torch.Tensor:
|
||||
q_out = SeqAllToAll4D.apply(ulysses_pg, q_in, 2, 1, use_sync)
|
||||
SeqAllToAll4D.apply(ulysses_pg, k_in, 2, 1, use_sync)
|
||||
SeqAllToAll4D.apply(ulysses_pg, v_in, 2, 1, use_sync)
|
||||
o_in = q_out # dummy
|
||||
return SeqAllToAll4D.apply(ulysses_pg, o_in, 1, 2, use_sync)
|
||||
|
||||
def uaa_comm_step(q_in: torch.Tensor, k_in: torch.Tensor, v_in: torch.Tensor) -> torch.Tensor:
|
||||
seq_lens = _all_gather_int(ulysses_pg, int(q_in.shape[1]), device=q_in.device)
|
||||
s_global = int(sum(seq_lens))
|
||||
if ring_world_size > 1:
|
||||
ring_s_globals = _all_gather_int(ring_pg, s_global, device=q_in.device)
|
||||
if len(set(ring_s_globals)) != 1:
|
||||
raise RuntimeError(f"Unexpected hybrid ring post-Ulysses seq_len mismatch: {ring_s_globals}.")
|
||||
|
||||
q_out, orig_head_cnt = _ulysses_all_to_all_any_qkv(ulysses_pg, q_in, seq_lens=seq_lens, use_sync=use_sync)
|
||||
_ulysses_all_to_all_any_qkv(ulysses_pg, k_in, seq_lens=seq_lens, use_sync=use_sync)
|
||||
_ulysses_all_to_all_any_qkv(ulysses_pg, v_in, seq_lens=seq_lens, use_sync=use_sync)
|
||||
o_in = q_out # dummy
|
||||
return _ulysses_all_to_all_any_o(
|
||||
ulysses_pg,
|
||||
o_in,
|
||||
seq_lens=seq_lens,
|
||||
local_seq_len=int(s_local),
|
||||
orig_head_cnt=int(orig_head_cnt),
|
||||
use_sync=use_sync,
|
||||
)
|
||||
|
||||
with torch.no_grad():
|
||||
# Warmup (strict)
|
||||
for _ in range(warmup_iters):
|
||||
_ = strict_comm_step(q, k, v)
|
||||
current_omni_platform.synchronize()
|
||||
|
||||
# Timed (strict)
|
||||
t0 = torch.cuda.Event(enable_timing=True)
|
||||
t1 = torch.cuda.Event(enable_timing=True)
|
||||
t0.record()
|
||||
for _ in range(iters):
|
||||
_ = strict_comm_step(q, k, v)
|
||||
t1.record()
|
||||
current_omni_platform.synchronize()
|
||||
strict_ms = float(t0.elapsed_time(t1)) / float(iters)
|
||||
|
||||
# Warmup (UAA)
|
||||
for _ in range(warmup_iters):
|
||||
_ = uaa_comm_step(q, k, v)
|
||||
current_omni_platform.synchronize()
|
||||
|
||||
# Timed (UAA)
|
||||
u0 = torch.cuda.Event(enable_timing=True)
|
||||
u1 = torch.cuda.Event(enable_timing=True)
|
||||
u0.record()
|
||||
for _ in range(iters):
|
||||
_ = uaa_comm_step(q, k, v)
|
||||
u1.record()
|
||||
current_omni_platform.synchronize()
|
||||
uaa_ms = float(u0.elapsed_time(u1)) / float(iters)
|
||||
|
||||
# Reduce across ranks (use worst-rank to be conservative).
|
||||
strict_ms_max = _max_all_reduce(dist.group.WORLD, strict_ms, device=device)
|
||||
uaa_ms_max = _max_all_reduce(dist.group.WORLD, uaa_ms, device=device)
|
||||
ratio = (uaa_ms_max / strict_ms_max) if strict_ms_max > 0 else float("inf")
|
||||
|
||||
# Approx bytes moved per iteration per-rank (send+recv) for 4x all-to-all.
|
||||
elem_size = torch.tensor([], dtype=dtype).element_size()
|
||||
per_a2a_bytes = int(q.numel()) * elem_size * 2
|
||||
comm_bytes = int(4 * per_a2a_bytes)
|
||||
strict_gbps = (comm_bytes / (strict_ms_max / 1000.0)) / 1e9 if strict_ms_max > 0 else 0.0
|
||||
uaa_gbps = (comm_bytes / (uaa_ms_max / 1000.0)) / 1e9 if uaa_ms_max > 0 else 0.0
|
||||
|
||||
if dist.get_rank() == 0:
|
||||
payload = {
|
||||
"name": "ulysses_uaa_comm_perf",
|
||||
"world_size": int(world_size),
|
||||
"ulysses_degree": int(ulysses_degree),
|
||||
"ring_degree": int(ring_degree),
|
||||
"ulysses_world_size": int(ulysses_world_size),
|
||||
"ring_world_size": int(ring_world_size),
|
||||
"shape": {
|
||||
"bsz": int(bsz),
|
||||
"s_local": int(s_local),
|
||||
"head_cnt": int(head_cnt),
|
||||
"head_dim": int(head_dim),
|
||||
},
|
||||
"dtype": str(dtype),
|
||||
"iters": int(iters),
|
||||
"strict_ms_per_iter_max": float(strict_ms_max),
|
||||
"uaa_ms_per_iter_max": float(uaa_ms_max),
|
||||
"uaa_over_strict_ratio": float(ratio),
|
||||
"comm_bytes_per_iter_per_rank": int(comm_bytes),
|
||||
"strict_effective_gbps_per_rank": float(strict_gbps),
|
||||
"uaa_effective_gbps_per_rank": float(uaa_gbps),
|
||||
}
|
||||
print(f"UAA_COMM_PERF_JSON={payload}")
|
||||
|
||||
# Loose bound: we only want to catch severe regressions. The printed JSON
|
||||
# payload is used for monitoring smaller changes, while this threshold is
|
||||
# intentionally generous to avoid flakiness across GPU types/drivers.
|
||||
max_uaa_ms_per_iter = 10.0
|
||||
assert uaa_ms_max < max_uaa_ms_per_iter, (
|
||||
f"UAA comm too slow: uaa={uaa_ms_max:.3f}ms/iter, strict={strict_ms_max:.3f}ms/iter "
|
||||
f"(cap={max_uaa_ms_per_iter:.1f}ms/iter)."
|
||||
)
|
||||
assert ratio < 3.0, f"UAA comm overhead too high: ratio={ratio:.3f}x (strict={strict_ms_max:.3f}ms)."
|
||||
finally:
|
||||
destroy_distributed_env()
|
||||
Reference in new issue
Block a user