[Diffusion][Bug-Fix]: Fix Ulysses SP Accurary Error (#377)

Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
Didan DengandJiangyun Zhu authored and GitHub committed 2025-12-22 17:48:39 +08:00
1 parent b06a5148f6
commit ee2b78e826
12 files changed
+747 -199

No files matched your search

@@ -0,0 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Parallel attention strategies.
This package provides **communication / resharding strategies** for attention,
orthogonal to the **attention kernel backend** (SDPA/Flash/Sage).
The goal is to keep `vllm_omni.diffusion.attention.layer.Attention` small and
extensible: adding a new parallelism method should not require editing the core
Attention module, only adding a new strategy and selecting it in the factory.
"""
from .base import NoParallelAttention, ParallelAttentionContext, ParallelAttentionStrategy
from .factory import build_parallel_attention_strategy
__all__ = [
"ParallelAttentionStrategy",
"ParallelAttentionContext",
"NoParallelAttention",
"build_parallel_attention_strategy",
]
@@ -0,0 +1,82 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from dataclasses import dataclass
from typing import Protocol
import torch
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
@dataclass(frozen=True, slots=True)
class ParallelAttentionContext:
"""Opaque per-forward context returned by a parallel strategy.
Strategies may stash whatever they need here to finish post-processing after
the attention kernel runs (e.g. reverse resharding, slicing metadata, etc.).
"""
name: str
class ParallelAttentionStrategy(Protocol):
"""Pluggable strategy for parallel attention communication/resharding.
This is intentionally orthogonal to the attention *kernel* backend.
The kernel backend implements `AttentionImpl.forward()` for a given device,
while the parallel strategy implements how Q/K/V and outputs are sharded /
communicated across ranks.
"""
@property
def enabled(self) -> bool: ...
@property
def name(self) -> str: ...
def pre_attention(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: AttentionMetadata | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, AttentionMetadata | None, ParallelAttentionContext | None]:
"""Runs before the attention kernel.
Returns possibly transformed Q/K/V and metadata, and an optional context
for `post_attention`.
"""
def post_attention(
self,
attn_output: torch.Tensor,
ctx: ParallelAttentionContext | None,
) -> torch.Tensor:
"""Runs after the attention kernel."""
class NoParallelAttention:
"""Default strategy: do nothing (single device / no SP)."""
@property
def enabled(self) -> bool:
return False
@property
def name(self) -> str:
return "none"
def pre_attention(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: AttentionMetadata | None,
):
return query, key, value, attn_metadata, None
def post_attention(self, attn_output: torch.Tensor, ctx: ParallelAttentionContext | None) -> torch.Tensor:
return attn_output
@@ -0,0 +1,45 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from vllm_omni.diffusion.attention.parallel.base import NoParallelAttention, ParallelAttentionStrategy
from vllm_omni.diffusion.attention.parallel.ulysses import UlyssesParallelAttention
from vllm_omni.diffusion.data import get_current_omni_diffusion_config
from vllm_omni.diffusion.distributed.parallel_state import get_sp_group
def build_parallel_attention_strategy(
*,
scatter_idx: int,
gather_idx: int,
use_sync: bool,
) -> ParallelAttentionStrategy:
"""Select a parallel attention strategy based on current diffusion config.
Design principle:
- Attention kernel backend selection remains in `attention/selector.py`.
- Parallel attention selection is handled here, based on distributed config
and initialized process groups.
"""
try:
cfg = get_current_omni_diffusion_config()
p = cfg.parallel_config
except Exception:
return NoParallelAttention()
# Current implementation supports Ulysses sequence-parallel attention.
# We intentionally do NOT infer ring attention from ring_degree here yet.
if getattr(p, "ulysses_degree", 1) > 1:
try:
sp_group = get_sp_group()
except Exception:
return NoParallelAttention()
return UlyssesParallelAttention(
sp_group=sp_group,
scatter_idx=scatter_idx,
gather_idx=gather_idx,
use_sync=use_sync,
)
return NoParallelAttention()
@@ -0,0 +1,135 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from dataclasses import dataclass
import torch
import torch.distributed as dist
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.parallel.base import ParallelAttentionContext
from vllm_omni.diffusion.distributed.comm import SeqAllToAll4D
from vllm_omni.diffusion.distributed.group_coordinator import SequenceParallelGroupCoordinator
@dataclass(frozen=True, slots=True)
class _UlyssesCtx(ParallelAttentionContext):
"""Per-forward context for Ulysses sequence-parallel attention."""
ulysses_pg: dist.ProcessGroup
scatter_idx: int
gather_idx: int
use_sync: bool
class UlyssesParallelAttention:
"""Ulysses sequence-parallel strategy (all-to-all over seq/head dims).
This preserves the semantics previously implemented in
`Attention._forward_ulysses`:
- If `AttentionMetadata.joint_*` is provided, joint_query is concatenated to
query *before* all-to-all; joint_key/value are concatenated *after* all-to-all.
- joint_key/value are assumed to be replicated across SP ranks and are sliced
by ulysses head rank before concatenation.
"""
def __init__(
self,
sp_group: SequenceParallelGroupCoordinator,
scatter_idx: int,
gather_idx: int,
use_sync: bool,
) -> None:
self._sp_group = sp_group
self._ulysses_pg = sp_group.ulysses_group
self._scatter_idx = scatter_idx
self._gather_idx = gather_idx
self._use_sync = use_sync
@property
def enabled(self) -> bool:
return True
@property
def name(self) -> str:
return "ulysses"
def pre_attention(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: AttentionMetadata | None,
):
joint_tensor_query = joint_tensor_key = joint_tensor_value = None
joint_strategy = None
if attn_metadata is not None:
joint_tensor_query = attn_metadata.joint_query
joint_tensor_key = attn_metadata.joint_key
joint_tensor_value = attn_metadata.joint_value
joint_strategy = attn_metadata.joint_strategy
is_joint = False
if joint_tensor_query is not None and joint_tensor_key is not None and joint_tensor_value is not None:
supported_joint_strategy = ["front", "rear"]
if joint_strategy not in supported_joint_strategy:
raise ValueError(
f"joint_strategy: {joint_strategy} not supported."
f" supported joint strategy: {supported_joint_strategy}"
)
if joint_strategy == "rear":
query = torch.cat([query, joint_tensor_query], dim=1)
else:
query = torch.cat([joint_tensor_query, query], dim=1)
is_joint = True
elif joint_tensor_query is None and joint_tensor_key is None and joint_tensor_value is None:
pass
else:
raise ValueError("joint_query, joint_key, and joint_value should be None or not None simultaneously.")
if is_joint:
# Slice joint key/value heads for this ulysses rank.
ulysses_world_size = self._sp_group.ulysses_world_size
ulysses_rank = self._sp_group.ulysses_rank
attn_heads_per_ulysses_rank = joint_tensor_key.shape[-2] // ulysses_world_size
joint_tensor_key = joint_tensor_key[
...,
attn_heads_per_ulysses_rank * ulysses_rank : attn_heads_per_ulysses_rank * (ulysses_rank + 1),
:,
]
joint_tensor_value = joint_tensor_value[
...,
attn_heads_per_ulysses_rank * ulysses_rank : attn_heads_per_ulysses_rank * (ulysses_rank + 1),
:,
]
# (bs, seq_len/P, head_cnt, head_size) -> (bs, seq_len, head_cnt/P, head_size)
query = SeqAllToAll4D.apply(self._ulysses_pg, query, self._scatter_idx, self._gather_idx, self._use_sync)
key = SeqAllToAll4D.apply(self._ulysses_pg, key, self._scatter_idx, self._gather_idx, self._use_sync)
value = SeqAllToAll4D.apply(self._ulysses_pg, value, self._scatter_idx, self._gather_idx, self._use_sync)
if is_joint:
# Concatenate joint key/value after all-to-all (matches previous implementation).
if joint_strategy == "front":
key = torch.cat([joint_tensor_key, key], dim=1)
value = torch.cat([joint_tensor_value, value], dim=1)
else: # "rear"
key = torch.cat([key, joint_tensor_key], dim=1)
value = torch.cat([value, joint_tensor_value], dim=1)
ctx = _UlyssesCtx(
name=self.name,
ulysses_pg=self._ulysses_pg,
scatter_idx=self._scatter_idx,
gather_idx=self._gather_idx,
use_sync=self._use_sync,
)
return query, key, value, attn_metadata, ctx
def post_attention(self, attn_output: torch.Tensor, ctx: ParallelAttentionContext | None) -> torch.Tensor:
assert isinstance(ctx, _UlyssesCtx), f"Unexpected ctx type: {type(ctx)!r}"
# Reverse: (bs, seq_len, head_cnt/P, head_size) -> (bs, seq_len/P, head_cnt, head_size)
return SeqAllToAll4D.apply(ctx.ulysses_pg, attn_output, ctx.gather_idx, ctx.scatter_idx, ctx.use_sync)