[Feature][Diffusion] Add BOOGU Image sequence parallel support (#5718)

Signed-off-by: Xenoryn <timberh2000@outlook.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-09-25 16:18:23 +08:00
1 parent e5613794ca
commit 0c7042d0d5
12 files changed
+1461 -68

No files matched your search

+114 -3
View File
@@ -5,13 +5,16 @@ from __future__ import annotations
import os
import tempfile
from contextlib import AbstractContextManager
from types import SimpleNamespace
import numpy as np
import pytest
import torch
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import get_distributed_init_method
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.attention.parallel.ulysses import UlyssesParallelAttention
from vllm_omni.diffusion.config import set_current_diffusion_config
@@ -578,6 +581,7 @@ def _run_attention_case(
num_kv_heads: int | None = None,
has_joint: bool = False,
joint_strategy: str = "front",
mask_layout: str | None = None,
) -> None:
device = torch.device(f"{current_omni_platform.device_type}:{local_rank}")
current_omni_platform.set_device(device)
@@ -620,6 +624,7 @@ def _run_attention_case(
q_full = torch.from_numpy(payload["q"]).to(device=device)
k_full = torch.from_numpy(payload["k"]).to(device=device)
v_full = torch.from_numpy(payload["v"]).to(device=device)
mask_full = torch.from_numpy(payload["mask"]).to(device=device) if "mask" in payload else None
if has_joint:
joint_q = torch.from_numpy(payload["joint_q"]).to(device=device)
joint_k = torch.from_numpy(payload["joint_k"]).to(device=device)
@@ -640,6 +645,7 @@ def _run_attention_case(
v = torch.cat([v_full, joint_v], dim=1)
else:
q, k, v = q_full, k_full, v_full
mask = mask_full
else:
if split_sizes is None:
# NOTE: torch.chunk may return fewer than `world_size` chunks for some
@@ -659,14 +665,18 @@ def _run_attention_case(
q = torch.split(q_full, split_sizes, dim=1)[local_rank].contiguous()
k = torch.split(k_full, split_sizes, dim=1)[local_rank].contiguous()
v = torch.split(v_full, split_sizes, dim=1)[local_rank].contiguous()
if mask_full is None:
mask = None
elif mask_layout == "global":
mask = mask_full
else:
raise ValueError("mask_layout must be 'global' when a mask is provided.")
# The Attention layer only enables SP communication when ForwardContext.sp_active is True.
# In production this is managed by SequenceParallelSplitHook/GatherHook, but here we
# shard manually for testing.
get_forward_context()._sp_shard_depth = 1
if has_joint:
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
attn_metadata = AttentionMetadata(
joint_query=joint_q,
joint_key=joint_k,
@@ -682,6 +692,8 @@ def _run_attention_case(
raise ValueError(f"Invalid sdp_kernel_mode: {sdp_kernel_mode!r}")
with sdp_ctx:
if mask is not None:
attn_metadata = AttentionMetadata(attn_mask=mask)
out_local = attn(q, k, v, attn_metadata=attn_metadata).contiguous()
if world_size == 1:
@@ -908,7 +920,7 @@ def _run_contract_skip_case(
ctx = get_forward_context()
ctx._sp_shard_depth = 1
def make_sdp_ctx() -> object:
def make_sdp_ctx() -> AbstractContextManager[None]:
# Fresh instance per phase: the @contextmanager instance releases
# its state on first exit and cannot be re-entered.
return torch.backends.cuda.sdp_kernel(enable_flash=False, enable_mem_efficient=False, enable_math=True)
@@ -1011,6 +1023,105 @@ def test_ulysses_uaa_contract_skips_length_gather_matches_slow_path(num_heads: i
pass
@pytest.mark.core_model
@pytest.mark.parametrize("mask_layout", ["global"])
@hardware_test(res={"cuda": "L4"}, num_cards=2)
def test_ulysses_uaa_2d_mask_layout_matches_baseline(
mask_layout: str,
) -> None:
world_size = 2
batch_size = 2
seq_len = 5
split_sizes = [2, 3]
num_heads = 3
head_size = 8
base_init_method = get_distributed_init_method()
sp_init_method = get_distributed_init_method()
with tempfile.NamedTemporaryFile(delete=False, suffix=".npz") as f_in:
input_file = f_in.name
with tempfile.NamedTemporaryFile(delete=False, suffix=".npy") as f_base:
baseline_file = f_base.name
with tempfile.NamedTemporaryFile(delete=False, suffix=".npy") as f_sp:
sp_file = f_sp.name
try:
torch.manual_seed(0)
q = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
k = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
v = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
mask = torch.tensor(
[
[True, True, True, True, False],
[True, True, True, False, False],
],
dtype=torch.bool,
)
np.savez(
input_file,
q=q.numpy(),
k=k.numpy(),
v=v.numpy(),
mask=mask.numpy(),
)
# Baseline (no SP), mask is used verbatim.
torch.multiprocessing.spawn(
_run_attention_case,
args=(
1,
base_init_method,
input_file,
baseline_file,
num_heads,
head_size,
1,
"strict",
1,
None,
"math",
None,
False,
"front",
"global",
),
nprocs=1,
)
# SP (Ulysses-P with UAA), mask is shared globally across ranks.
torch.multiprocessing.spawn(
_run_attention_case,
args=(
world_size,
sp_init_method,
input_file,
sp_file,
num_heads,
head_size,
world_size,
"advanced_uaa",
1,
split_sizes,
"math",
None,
False,
"front",
mask_layout,
),
nprocs=world_size,
)
baseline = torch.from_numpy(np.load(baseline_file, allow_pickle=False))
sp = torch.from_numpy(np.load(sp_file, allow_pickle=False))
torch.testing.assert_close(sp, baseline, atol=1e-5, rtol=1e-5)
finally:
for path in (input_file, baseline_file, sp_file):
try:
os.remove(path)
except OSError:
pass
@pytest.mark.parametrize(
"num_heads,num_kv_heads,joint_len",
[