[Feature][Diffusion] Add BOOGU Image sequence parallel support (#5718)
Signed-off-by: Xenoryn <timberh2000@outlook.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
12 files changed
+1461
-68
No files matched your search
@@ -5,13 +5,16 @@ from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from contextlib import AbstractContextManager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import get_distributed_init_method
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.attention.parallel.ulysses import UlyssesParallelAttention
|
||||
from vllm_omni.diffusion.config import set_current_diffusion_config
|
||||
@@ -578,6 +581,7 @@ def _run_attention_case(
|
||||
num_kv_heads: int | None = None,
|
||||
has_joint: bool = False,
|
||||
joint_strategy: str = "front",
|
||||
mask_layout: str | None = None,
|
||||
) -> None:
|
||||
device = torch.device(f"{current_omni_platform.device_type}:{local_rank}")
|
||||
current_omni_platform.set_device(device)
|
||||
@@ -620,6 +624,7 @@ def _run_attention_case(
|
||||
q_full = torch.from_numpy(payload["q"]).to(device=device)
|
||||
k_full = torch.from_numpy(payload["k"]).to(device=device)
|
||||
v_full = torch.from_numpy(payload["v"]).to(device=device)
|
||||
mask_full = torch.from_numpy(payload["mask"]).to(device=device) if "mask" in payload else None
|
||||
if has_joint:
|
||||
joint_q = torch.from_numpy(payload["joint_q"]).to(device=device)
|
||||
joint_k = torch.from_numpy(payload["joint_k"]).to(device=device)
|
||||
@@ -640,6 +645,7 @@ def _run_attention_case(
|
||||
v = torch.cat([v_full, joint_v], dim=1)
|
||||
else:
|
||||
q, k, v = q_full, k_full, v_full
|
||||
mask = mask_full
|
||||
else:
|
||||
if split_sizes is None:
|
||||
# NOTE: torch.chunk may return fewer than `world_size` chunks for some
|
||||
@@ -659,14 +665,18 @@ def _run_attention_case(
|
||||
q = torch.split(q_full, split_sizes, dim=1)[local_rank].contiguous()
|
||||
k = torch.split(k_full, split_sizes, dim=1)[local_rank].contiguous()
|
||||
v = torch.split(v_full, split_sizes, dim=1)[local_rank].contiguous()
|
||||
if mask_full is None:
|
||||
mask = None
|
||||
elif mask_layout == "global":
|
||||
mask = mask_full
|
||||
else:
|
||||
raise ValueError("mask_layout must be 'global' when a mask is provided.")
|
||||
# The Attention layer only enables SP communication when ForwardContext.sp_active is True.
|
||||
# In production this is managed by SequenceParallelSplitHook/GatherHook, but here we
|
||||
# shard manually for testing.
|
||||
get_forward_context()._sp_shard_depth = 1
|
||||
|
||||
if has_joint:
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
|
||||
attn_metadata = AttentionMetadata(
|
||||
joint_query=joint_q,
|
||||
joint_key=joint_k,
|
||||
@@ -682,6 +692,8 @@ def _run_attention_case(
|
||||
raise ValueError(f"Invalid sdp_kernel_mode: {sdp_kernel_mode!r}")
|
||||
|
||||
with sdp_ctx:
|
||||
if mask is not None:
|
||||
attn_metadata = AttentionMetadata(attn_mask=mask)
|
||||
out_local = attn(q, k, v, attn_metadata=attn_metadata).contiguous()
|
||||
|
||||
if world_size == 1:
|
||||
@@ -908,7 +920,7 @@ def _run_contract_skip_case(
|
||||
ctx = get_forward_context()
|
||||
ctx._sp_shard_depth = 1
|
||||
|
||||
def make_sdp_ctx() -> object:
|
||||
def make_sdp_ctx() -> AbstractContextManager[None]:
|
||||
# Fresh instance per phase: the @contextmanager instance releases
|
||||
# its state on first exit and cannot be re-entered.
|
||||
return torch.backends.cuda.sdp_kernel(enable_flash=False, enable_mem_efficient=False, enable_math=True)
|
||||
@@ -1011,6 +1023,105 @@ def test_ulysses_uaa_contract_skips_length_gather_matches_slow_path(num_heads: i
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("mask_layout", ["global"])
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=2)
|
||||
def test_ulysses_uaa_2d_mask_layout_matches_baseline(
|
||||
mask_layout: str,
|
||||
) -> None:
|
||||
world_size = 2
|
||||
batch_size = 2
|
||||
seq_len = 5
|
||||
split_sizes = [2, 3]
|
||||
num_heads = 3
|
||||
head_size = 8
|
||||
|
||||
base_init_method = get_distributed_init_method()
|
||||
sp_init_method = get_distributed_init_method()
|
||||
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".npz") as f_in:
|
||||
input_file = f_in.name
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".npy") as f_base:
|
||||
baseline_file = f_base.name
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".npy") as f_sp:
|
||||
sp_file = f_sp.name
|
||||
|
||||
try:
|
||||
torch.manual_seed(0)
|
||||
q = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
|
||||
k = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
|
||||
v = torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32)
|
||||
mask = torch.tensor(
|
||||
[
|
||||
[True, True, True, True, False],
|
||||
[True, True, True, False, False],
|
||||
],
|
||||
dtype=torch.bool,
|
||||
)
|
||||
np.savez(
|
||||
input_file,
|
||||
q=q.numpy(),
|
||||
k=k.numpy(),
|
||||
v=v.numpy(),
|
||||
mask=mask.numpy(),
|
||||
)
|
||||
|
||||
# Baseline (no SP), mask is used verbatim.
|
||||
torch.multiprocessing.spawn(
|
||||
_run_attention_case,
|
||||
args=(
|
||||
1,
|
||||
base_init_method,
|
||||
input_file,
|
||||
baseline_file,
|
||||
num_heads,
|
||||
head_size,
|
||||
1,
|
||||
"strict",
|
||||
1,
|
||||
None,
|
||||
"math",
|
||||
None,
|
||||
False,
|
||||
"front",
|
||||
"global",
|
||||
),
|
||||
nprocs=1,
|
||||
)
|
||||
# SP (Ulysses-P with UAA), mask is shared globally across ranks.
|
||||
torch.multiprocessing.spawn(
|
||||
_run_attention_case,
|
||||
args=(
|
||||
world_size,
|
||||
sp_init_method,
|
||||
input_file,
|
||||
sp_file,
|
||||
num_heads,
|
||||
head_size,
|
||||
world_size,
|
||||
"advanced_uaa",
|
||||
1,
|
||||
split_sizes,
|
||||
"math",
|
||||
None,
|
||||
False,
|
||||
"front",
|
||||
mask_layout,
|
||||
),
|
||||
nprocs=world_size,
|
||||
)
|
||||
|
||||
baseline = torch.from_numpy(np.load(baseline_file, allow_pickle=False))
|
||||
sp = torch.from_numpy(np.load(sp_file, allow_pickle=False))
|
||||
torch.testing.assert_close(sp, baseline, atol=1e-5, rtol=1e-5)
|
||||
finally:
|
||||
for path in (input_file, baseline_file, sp_file):
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_heads,num_kv_heads,joint_len",
|
||||
[
|
||||
|
||||
Reference in new issue
Block a user