[Feature] Add AllGather-KV sequence-parallel attention str… (#4968)
Signed-off-by: zhuhh97 <zhuhonghan@huawei.com> Signed-off-by: dph97 <nenbaying@163.com> Signed-off-by: zhuhh97 <zhh@unknown> Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com> Co-authored-by: dengyunyang <584797741@qq.com>
This commit is contained in:
33 files changed
+1181
-135
No files matched your search
@@ -0,0 +1,182 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import QueryRange
|
||||
from vllm_omni.diffusion.attention.parallel.base import ParallelAttentionContext
|
||||
from vllm_omni.diffusion.distributed.group_coordinator import (
|
||||
SequenceParallelGroupCoordinator,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _AllGatherKVCtx(ParallelAttentionContext):
|
||||
pass
|
||||
|
||||
|
||||
class AllGatherKVParallelAttention:
|
||||
"""Compute local Q against AllGathered K/V."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sp_group: SequenceParallelGroupCoordinator,
|
||||
) -> None:
|
||||
self._sp_group = sp_group
|
||||
self._allgather_group = sp_group.allgather_group
|
||||
self._sp_size = sp_group.allgather_world_size
|
||||
self._sp_rank = sp_group.allgather_rank
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "allgather_kv"
|
||||
|
||||
def pre_attention(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata | None,
|
||||
):
|
||||
joint_q = joint_k = joint_v = None
|
||||
joint_strategy = "front"
|
||||
if attn_metadata is not None:
|
||||
joint_q = attn_metadata.joint_query
|
||||
joint_k = attn_metadata.joint_key
|
||||
joint_v = attn_metadata.joint_value
|
||||
joint_strategy = attn_metadata.joint_strategy or "front"
|
||||
if joint_strategy not in {"front", "rear"}:
|
||||
raise ValueError(f"Unsupported joint_strategy: {joint_strategy!r}")
|
||||
|
||||
k_img_full = self._sp_group.all_gather(key, dim=1, group=self._allgather_group)
|
||||
v_img_full = self._sp_group.all_gather(value, dim=1, group=self._allgather_group)
|
||||
|
||||
if joint_k is not None:
|
||||
if joint_k.shape[2] != key.shape[2]:
|
||||
raise ValueError(
|
||||
"AllGather-KV SP expects joint_k to be GQA-compressed with the "
|
||||
f"same num_kv_heads as the image shard, got joint_k.heads="
|
||||
f"{joint_k.shape[2]} vs key.heads={key.shape[2]}."
|
||||
)
|
||||
if joint_strategy == "front":
|
||||
k_full = torch.cat([joint_k, k_img_full], dim=1)
|
||||
v_full = torch.cat([joint_v, v_img_full], dim=1)
|
||||
else:
|
||||
k_full = torch.cat([k_img_full, joint_k], dim=1)
|
||||
v_full = torch.cat([v_img_full, joint_v], dim=1)
|
||||
else:
|
||||
k_full, v_full = k_img_full, v_img_full
|
||||
|
||||
joint_len = joint_q.shape[1] if joint_q is not None else 0
|
||||
logical_q_full_len = joint_len + k_img_full.shape[1]
|
||||
query_global_offset = k_full.shape[1] - logical_q_full_len
|
||||
if query_global_offset < 0:
|
||||
raise ValueError(
|
||||
"AllGather-KV SP global K length is shorter than the logical Q length: "
|
||||
f"k_len={k_full.shape[1]}, logical_q_len={logical_q_full_len}."
|
||||
)
|
||||
|
||||
if joint_q is not None:
|
||||
if joint_strategy == "front":
|
||||
q_local = torch.cat([joint_q, query], dim=1)
|
||||
else:
|
||||
q_local = torch.cat([query, joint_q], dim=1)
|
||||
else:
|
||||
q_local = query
|
||||
|
||||
attn_metadata = self._slice_attn_metadata_for_local_query(
|
||||
attn_metadata,
|
||||
q_local_len=q_local.shape[1],
|
||||
img_seq_local=query.shape[1],
|
||||
img_seq_full=k_img_full.shape[1],
|
||||
joint_len=joint_len,
|
||||
joint_strategy=joint_strategy,
|
||||
query_global_offset=query_global_offset,
|
||||
)
|
||||
|
||||
return q_local, k_full, v_full, attn_metadata, _AllGatherKVCtx(name=self.name)
|
||||
|
||||
def _slice_attn_metadata_for_local_query(
|
||||
self,
|
||||
attn_metadata: AttentionMetadata | None,
|
||||
*,
|
||||
q_local_len: int,
|
||||
img_seq_local: int,
|
||||
img_seq_full: int,
|
||||
joint_len: int,
|
||||
joint_strategy: str,
|
||||
query_global_offset: int,
|
||||
) -> AttentionMetadata | None:
|
||||
"""Slice global query metadata for the local query shard."""
|
||||
if attn_metadata is None:
|
||||
return None
|
||||
|
||||
img_start = self._sp_rank * img_seq_local
|
||||
img_end = img_start + img_seq_local
|
||||
if img_end > img_seq_full:
|
||||
raise ValueError(
|
||||
"AllGather-KV SP local image query range exceeds gathered image length: "
|
||||
f"rank={self._sp_rank}, img_start={img_start}, img_end={img_end}, img_seq_full={img_seq_full}."
|
||||
)
|
||||
|
||||
if joint_strategy == "front":
|
||||
ranges = (
|
||||
*((QueryRange(0, joint_len, query_global_offset),) if joint_len else ()),
|
||||
QueryRange(joint_len, q_local_len, query_global_offset + joint_len + img_start),
|
||||
)
|
||||
else:
|
||||
ranges = (
|
||||
QueryRange(0, img_seq_local, query_global_offset + img_start),
|
||||
*((QueryRange(img_seq_local, q_local_len, query_global_offset + img_seq_full),) if joint_len else ()),
|
||||
)
|
||||
|
||||
if attn_metadata.attn_mask is None:
|
||||
return replace(attn_metadata, query_ranges=ranges)
|
||||
|
||||
mask = attn_metadata.attn_mask
|
||||
if mask.ndim != 4:
|
||||
return replace(attn_metadata, query_ranges=ranges)
|
||||
|
||||
if mask.shape[-2] == q_local_len:
|
||||
local_mask = mask
|
||||
else:
|
||||
q_full_len = joint_len + img_seq_full
|
||||
if mask.shape[-2] != q_full_len:
|
||||
raise ValueError(
|
||||
"AllGather-KV SP received an attention mask with incompatible Q length: "
|
||||
f"mask_q={mask.shape[-2]}, expected local_q={q_local_len} or full_q={q_full_len} "
|
||||
f"(joint_len={joint_len}, img_seq_local={img_seq_local}, img_seq_full={img_seq_full})."
|
||||
)
|
||||
|
||||
parts = [
|
||||
mask[..., local_start : local_start + (r.local_end - r.local_start), :]
|
||||
for r in ranges
|
||||
for local_start in (r.global_start - query_global_offset,)
|
||||
]
|
||||
local_mask = torch.cat(parts, dim=-2)
|
||||
|
||||
if local_mask.shape[-2] != q_local_len:
|
||||
raise ValueError(
|
||||
"AllGather-KV SP produced an attention mask with incompatible local Q length: "
|
||||
f"mask_q={local_mask.shape[-2]}, q_local={q_local_len}."
|
||||
)
|
||||
return replace(attn_metadata, attn_mask=local_mask.contiguous(), query_ranges=ranges)
|
||||
|
||||
def post_attention(
|
||||
self,
|
||||
attn_output: torch.Tensor,
|
||||
ctx: ParallelAttentionContext | None,
|
||||
) -> torch.Tensor:
|
||||
return attn_output
|
||||
@@ -5,10 +5,19 @@ from __future__ import annotations
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.attention.parallel.base import NoParallelAttention, ParallelAttentionStrategy
|
||||
from vllm_omni.diffusion.attention.parallel.allgather_kv import (
|
||||
AllGatherKVParallelAttention,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.parallel.base import (
|
||||
NoParallelAttention,
|
||||
ParallelAttentionStrategy,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.parallel.ring import RingParallelAttention
|
||||
from vllm_omni.diffusion.attention.parallel.ulysses import UlyssesParallelAttention
|
||||
from vllm_omni.diffusion.distributed.parallel_state import get_sequence_parallel_world_size, get_sp_group
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
get_sequence_parallel_world_size,
|
||||
get_sp_group,
|
||||
)
|
||||
from vllm_omni.diffusion.forward_context import get_forward_context
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -19,6 +28,7 @@ def build_parallel_attention_strategy(
|
||||
scatter_idx: int,
|
||||
gather_idx: int,
|
||||
use_sync: bool,
|
||||
causal: bool = False,
|
||||
) -> ParallelAttentionStrategy:
|
||||
"""Select a parallel attention strategy based on current diffusion config.
|
||||
|
||||
@@ -36,6 +46,7 @@ def build_parallel_attention_strategy(
|
||||
|
||||
ulysses_degree = getattr(p, "ulysses_degree", 1)
|
||||
ring_degree = getattr(p, "ring_degree", 1)
|
||||
allgather_degree = getattr(p, "allgather_degree", 1)
|
||||
|
||||
try:
|
||||
sp_group = get_sp_group()
|
||||
@@ -44,13 +55,26 @@ def build_parallel_attention_strategy(
|
||||
return NoParallelAttention()
|
||||
except Exception as e:
|
||||
# Log warning if SP is configured but group is not available
|
||||
if ulysses_degree > 1 or ring_degree > 1:
|
||||
if ulysses_degree > 1 or ring_degree > 1 or allgather_degree > 1:
|
||||
logger.warning(
|
||||
f"SP configured (ulysses={ulysses_degree}, ring={ring_degree}) but SP group not available: {e}. "
|
||||
f"SP configured (ulysses={ulysses_degree}, ring={ring_degree}, "
|
||||
f"allgather={allgather_degree}) but SP group not available: {e}. "
|
||||
f"Falling back to NoParallelAttention. This may cause incorrect results."
|
||||
)
|
||||
return NoParallelAttention()
|
||||
|
||||
if allgather_degree > 1:
|
||||
if causal:
|
||||
raise ValueError("AllGather-KV SP only supports non-causal attention.")
|
||||
if ulysses_degree > 1 or ring_degree > 1:
|
||||
raise ValueError(
|
||||
f"AllGather-KV SP is mutually exclusive with Ulysses/Ring in v1 "
|
||||
f"(got ulysses_degree={ulysses_degree}, ring_degree={ring_degree}, "
|
||||
f"allgather_degree={allgather_degree})."
|
||||
)
|
||||
logger.debug(f"Using AllGatherKVParallelAttention (allgather_degree={allgather_degree})")
|
||||
return AllGatherKVParallelAttention(sp_group=sp_group)
|
||||
|
||||
# Ulysses (or Hybrid Ulysses+Ring)
|
||||
if ulysses_degree > 1:
|
||||
logger.debug(f"Using UlyssesParallelAttention (ulysses_degree={ulysses_degree})")
|
||||
|
||||
Reference in new issue
Block a user