[Feature][Diffusion] Add BOOGU Image sequence parallel support (#5718)
Signed-off-by: Xenoryn <timberh2000@outlook.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
12 files changed
+1461
-68
No files matched your search
@@ -0,0 +1,317 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import socket
|
||||
import tempfile
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.config import DeviceConfig, VllmConfig, set_current_vllm_config
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.diffusion.config import set_current_diffusion_config
|
||||
from vllm_omni.diffusion.data import (
|
||||
DiffusionParallelConfig,
|
||||
OmniDiffusionConfig,
|
||||
TransformerConfig,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
destroy_distributed_env,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelConfig
|
||||
from vllm_omni.diffusion.forward_context import (
|
||||
get_forward_context,
|
||||
set_forward_context,
|
||||
)
|
||||
from vllm_omni.diffusion.hooks.sequence_parallel import apply_sequence_parallel
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.core_model,
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.parallel,
|
||||
]
|
||||
|
||||
|
||||
def _find_free_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
return int(sock.getsockname()[1])
|
||||
|
||||
|
||||
def _set_dist_env(rank: int, world_size: int, port: int) -> None:
|
||||
os.environ.update(
|
||||
{
|
||||
"RANK": str(rank),
|
||||
"LOCAL_RANK": str(rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "127.0.0.1",
|
||||
"MASTER_PORT": str(port),
|
||||
"DIFFUSION_ATTENTION_BACKEND": "TORCH_SDPA",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _transformer_config() -> TransformerConfig:
|
||||
return TransformerConfig.from_dict(
|
||||
{
|
||||
"patch_size": 2,
|
||||
"in_channels": 4,
|
||||
"hidden_size": 64,
|
||||
"num_layers": 2,
|
||||
"num_double_stream_layers": 1,
|
||||
"num_refiner_layers": 1,
|
||||
"num_attention_heads": 4,
|
||||
"num_kv_heads": 1,
|
||||
"multiple_of": 32,
|
||||
"norm_eps": 1e-5,
|
||||
"axes_dim_rope": [8, 4, 4],
|
||||
"axes_lens": [64, 32, 32],
|
||||
"instruction_feature_configs": {
|
||||
"instruction_feat_dim": 32,
|
||||
"reduce_type": "mean",
|
||||
"num_instruction_feature_layers": 1,
|
||||
},
|
||||
"prompt_tuning_configs": {"use_prompt_tuning": False},
|
||||
"timestep_scale": 1.0,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _od_config(world_size: int) -> OmniDiffusionConfig:
|
||||
return OmniDiffusionConfig(
|
||||
model="boogu-tiny",
|
||||
dtype=torch.float32,
|
||||
tf_model_config=_transformer_config(),
|
||||
parallel_config=DiffusionParallelConfig(
|
||||
sequence_parallel_size=world_size,
|
||||
ulysses_degree=world_size,
|
||||
ulysses_mode="advanced_uaa",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _checkpoint_name(name: str) -> str:
|
||||
if ".to_out." in name:
|
||||
name = name.replace(".to_out.", ".to_out.0.")
|
||||
for projection in (
|
||||
"img_to_q",
|
||||
"img_to_k",
|
||||
"img_to_v",
|
||||
"instruct_to_q",
|
||||
"instruct_to_k",
|
||||
"instruct_to_v",
|
||||
"instruct_out",
|
||||
"img_out",
|
||||
):
|
||||
token = f".img_instruct_attn.{projection}."
|
||||
if token in name:
|
||||
return name.replace(
|
||||
token,
|
||||
f".img_instruct_attn.processor.{projection}.",
|
||||
)
|
||||
return name
|
||||
|
||||
|
||||
def _random_checkpoint(model: torch.nn.Module) -> list[tuple[str, torch.Tensor]]:
|
||||
generator = torch.Generator(device=next(model.parameters()).device).manual_seed(2026)
|
||||
return [
|
||||
(
|
||||
_checkpoint_name(name),
|
||||
torch.empty_like(param).uniform_(
|
||||
-0.02,
|
||||
0.02,
|
||||
generator=generator,
|
||||
),
|
||||
)
|
||||
for name, param in model.named_parameters()
|
||||
]
|
||||
|
||||
|
||||
def _worker(
|
||||
rank: int,
|
||||
world_size: int,
|
||||
port: int,
|
||||
output_file: str,
|
||||
) -> None:
|
||||
device = torch.device(f"{current_omni_platform.device_type}:{rank}")
|
||||
current_omni_platform.set_device(device)
|
||||
_set_dist_env(rank, world_size, port)
|
||||
init_distributed_environment(world_size=world_size, rank=rank)
|
||||
initialize_model_parallel(
|
||||
data_parallel_size=1,
|
||||
cfg_parallel_size=1,
|
||||
sequence_parallel_size=world_size,
|
||||
ulysses_degree=world_size,
|
||||
ring_degree=1,
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
)
|
||||
|
||||
try:
|
||||
from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import (
|
||||
BooguImageDoubleStreamRotaryPosEmbed,
|
||||
BooguImageTransformer2DModel,
|
||||
)
|
||||
|
||||
od_config = _od_config(world_size)
|
||||
with (
|
||||
set_current_vllm_config(VllmConfig(device_config=DeviceConfig(device=str(device)))),
|
||||
set_forward_context(omni_diffusion_config=od_config),
|
||||
set_current_diffusion_config(od_config),
|
||||
):
|
||||
source = BooguImageTransformer2DModel(od_config).to(device)
|
||||
checkpoint = _random_checkpoint(source)
|
||||
del source
|
||||
|
||||
model = BooguImageTransformer2DModel(od_config).to(device)
|
||||
assert model.load_weights(checkpoint) == set(dict(model.named_parameters()))
|
||||
model.eval()
|
||||
|
||||
if world_size > 1:
|
||||
apply_sequence_parallel(
|
||||
model,
|
||||
SequenceParallelConfig(ulysses_degree=world_size),
|
||||
model._sp_plan,
|
||||
)
|
||||
get_forward_context().sp_plan_hooks_applied = True
|
||||
|
||||
generator = torch.Generator(device=device).manual_seed(17)
|
||||
latents = torch.randn(
|
||||
1,
|
||||
4,
|
||||
8,
|
||||
8,
|
||||
device=device,
|
||||
generator=generator,
|
||||
)
|
||||
timestep = torch.tensor([0.5], device=device)
|
||||
instruction = torch.randn(
|
||||
1,
|
||||
7,
|
||||
32,
|
||||
device=device,
|
||||
generator=generator,
|
||||
)
|
||||
instruction_mask = torch.ones(
|
||||
1,
|
||||
7,
|
||||
dtype=torch.bool,
|
||||
device=device,
|
||||
)
|
||||
ref_image = [
|
||||
[
|
||||
torch.randn(
|
||||
4,
|
||||
6,
|
||||
10,
|
||||
device=device,
|
||||
generator=generator,
|
||||
)
|
||||
]
|
||||
]
|
||||
# Two differently sized references: their packed segments straddle
|
||||
# the SP rank boundary, which is what the per-rank reference-refiner
|
||||
# mask has to reproduce.
|
||||
ref_images_multi = [
|
||||
[
|
||||
torch.randn(4, 6, 10, device=device, generator=generator),
|
||||
torch.randn(4, 4, 8, device=device, generator=generator),
|
||||
]
|
||||
]
|
||||
freqs_real = BooguImageDoubleStreamRotaryPosEmbed.get_freqs_real(
|
||||
model.axes_dim_rope,
|
||||
model.axes_lens,
|
||||
theta=10000,
|
||||
)
|
||||
|
||||
with torch.inference_mode():
|
||||
t2i = model(
|
||||
latents,
|
||||
timestep,
|
||||
instruction,
|
||||
freqs_real,
|
||||
instruction_mask,
|
||||
)
|
||||
edit = model(
|
||||
latents,
|
||||
timestep,
|
||||
instruction,
|
||||
freqs_real,
|
||||
instruction_mask,
|
||||
ref_image_hidden_states=ref_image,
|
||||
)
|
||||
edit_multi = model(
|
||||
latents,
|
||||
timestep,
|
||||
instruction,
|
||||
freqs_real,
|
||||
instruction_mask,
|
||||
ref_image_hidden_states=ref_images_multi,
|
||||
)
|
||||
|
||||
if rank == 0:
|
||||
np.savez(
|
||||
output_file,
|
||||
t2i=t2i.cpu().numpy(),
|
||||
edit=edit.cpu().numpy(),
|
||||
edit_multi=edit_multi.cpu().numpy(),
|
||||
)
|
||||
finally:
|
||||
destroy_distributed_env()
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=2)
|
||||
def test_boogu_sp_matches_single_gpu() -> None:
|
||||
if not current_omni_platform.is_cuda() or current_omni_platform.get_device_count() < 2:
|
||||
pytest.skip("BOOGU SP test requires two CUDA GPUs.")
|
||||
|
||||
output_files = []
|
||||
try:
|
||||
for _ in range(2):
|
||||
with tempfile.NamedTemporaryFile(
|
||||
delete=False,
|
||||
suffix=".npz",
|
||||
) as handle:
|
||||
output_files.append(handle.name)
|
||||
|
||||
torch.multiprocessing.spawn(
|
||||
_worker,
|
||||
args=(1, _find_free_port(), output_files[0]),
|
||||
nprocs=1,
|
||||
)
|
||||
torch.multiprocessing.spawn(
|
||||
_worker,
|
||||
args=(2, _find_free_port(), output_files[1]),
|
||||
nprocs=2,
|
||||
)
|
||||
|
||||
with (
|
||||
np.load(output_files[0], allow_pickle=False) as baseline,
|
||||
np.load(output_files[1], allow_pickle=False) as sp_output,
|
||||
):
|
||||
for key in ("t2i", "edit", "edit_multi"):
|
||||
# fp32 SDPA: the observed SP1-vs-SP2 gap is ~3e-8, so 2e-4
|
||||
# would also pass with the SP masking logic entirely removed
|
||||
# (that mutation measures ~5e-7). Mask correctness is pinned
|
||||
# structurally in tests/diffusion/models/boogu_image/
|
||||
# test_sp_layout.py; this bound just stays close to the real
|
||||
# reduction-order noise instead of hiding logic errors.
|
||||
np.testing.assert_allclose(
|
||||
sp_output[key],
|
||||
baseline[key],
|
||||
rtol=1e-5,
|
||||
atol=1e-5,
|
||||
)
|
||||
finally:
|
||||
for path in output_files:
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
Reference in new issue
Block a user