[Feature] Add AllGather-KV sequence-parallel attention str… (#4968)
Signed-off-by: zhuhh97 <zhuhonghan@huawei.com> Signed-off-by: dph97 <nenbaying@163.com> Signed-off-by: zhuhh97 <zhh@unknown> Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com> Co-authored-by: dengyunyang <584797741@qq.com>
This commit is contained in:
33 files changed
+1181
-135
No files matched your search
@@ -0,0 +1,40 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Runtime kernel-selection helpers for attention backends."""
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def can_sdpa_use_fused_gqa(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attention_mask: torch.Tensor | None,
|
||||
causal: bool,
|
||||
) -> bool:
|
||||
"""Return whether PyTorch SDPA can use a fused kernel for this GQA call.
|
||||
|
||||
This check is specific to the SDPA backend. Native GQA backends such as
|
||||
FlashAttention should keep compressed K/V heads and do not use this helper.
|
||||
"""
|
||||
if not query.is_cuda:
|
||||
return False
|
||||
|
||||
try:
|
||||
params = torch.backends.cuda.SDPAParams(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attention_mask,
|
||||
0.0,
|
||||
causal,
|
||||
True,
|
||||
)
|
||||
can_use_flash = torch.backends.cuda.can_use_flash_attention(params)
|
||||
can_use_cudnn = getattr(torch.backends.cuda, "can_use_cudnn_attention", None)
|
||||
return can_use_flash or (can_use_cudnn is not None and can_use_cudnn(params))
|
||||
except (AttributeError, RuntimeError, TypeError):
|
||||
# Older PyTorch versions may not expose GQA-aware SDPA capability checks.
|
||||
# Returning False lets the SDPA backend expand K/V and use the mature
|
||||
# equal-head fused-kernel path instead.
|
||||
return False
|
||||
@@ -9,17 +9,24 @@ follows FlashAttention's bottom-right convention (``K[:e]`` is attended by
|
||||
|
||||
Per segment:
|
||||
- causal segment ``[s, e)``: ``attn(Q[:, s:e], K[:, :e], V[:, :e], causal=True)``
|
||||
- full-attn span ``[a, e)``: ``attn(Q[:, a:e], K[:, :e], V[:, :e], causal=False)``
|
||||
- full-attn span ``[a, b)`` intersecting the query range at ``[s, e)``:
|
||||
``attn(Q[:, s:e], K[:, :b], V[:, :b], causal=False)``
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal, NamedTuple
|
||||
from typing import TYPE_CHECKING, Literal, NamedTuple
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.diffusion.attention.backends.abstract import QueryRange
|
||||
|
||||
|
||||
class Segment(NamedTuple):
|
||||
start: int
|
||||
end: int
|
||||
q_start: int
|
||||
q_end: int
|
||||
kv_end: int
|
||||
mode: Literal["causal", "full"]
|
||||
|
||||
|
||||
@@ -30,30 +37,33 @@ def build_segments(full_attn_spans, query_offset, query_len):
|
||||
query_len: length of the query
|
||||
|
||||
return:
|
||||
List[Segment] in global coordinates, clipped to [query_offset, query_offset + query_len)
|
||||
List[Segment] in global coordinates, clipped to
|
||||
[query_offset, query_offset + query_len). Full-attention segments retain
|
||||
the original span end as kv_end so a local query shard can attend past
|
||||
its own boundary.
|
||||
"""
|
||||
q_start = query_offset
|
||||
q_end = query_offset + query_len
|
||||
|
||||
segs: list[Segment] = []
|
||||
segments: list[Segment] = []
|
||||
cur = q_start
|
||||
|
||||
for a, e in full_attn_spans:
|
||||
for span_start, span_end in full_attn_spans:
|
||||
# clip span to query range
|
||||
a_clipped = max(a, q_start)
|
||||
e_clipped = min(e, q_end)
|
||||
if a_clipped >= e_clipped:
|
||||
overlap_start = max(span_start, q_start)
|
||||
overlap_end = min(span_end, q_end)
|
||||
if overlap_start >= overlap_end:
|
||||
continue
|
||||
|
||||
if cur < a_clipped:
|
||||
segs.append(Segment(cur, a_clipped, "causal"))
|
||||
segs.append(Segment(a_clipped, e_clipped, "full"))
|
||||
cur = e_clipped
|
||||
if cur < overlap_start:
|
||||
segments.append(Segment(cur, overlap_start, overlap_start, "causal"))
|
||||
segments.append(Segment(overlap_start, overlap_end, span_end, "full"))
|
||||
cur = overlap_end
|
||||
|
||||
if cur < q_end:
|
||||
segs.append(Segment(cur, q_end, "causal"))
|
||||
segments.append(Segment(cur, q_end, q_end, "causal"))
|
||||
|
||||
return segs
|
||||
return segments
|
||||
|
||||
|
||||
def _check_homogeneous(
|
||||
@@ -76,23 +86,40 @@ def piecewise_attn(
|
||||
full_attn_spans: list[list[tuple[int, int]]],
|
||||
softmax_scale: float,
|
||||
attn_func,
|
||||
query_ranges: tuple[QueryRange, ...] | None = None,
|
||||
):
|
||||
B, Sq, H, D = query.shape
|
||||
_check_homogeneous(full_attn_spans)
|
||||
|
||||
query_offset = key.shape[1] - Sq
|
||||
spans = full_attn_spans[0]
|
||||
out = query.new_zeros(B, Sq, H, D)
|
||||
if query_ranges is None:
|
||||
query_len = query.shape[1]
|
||||
ranges = ((0, query_len, key.shape[1] - query_len),)
|
||||
else:
|
||||
ranges = tuple((r.local_start, r.local_end, r.global_start) for r in query_ranges)
|
||||
|
||||
for s, e, mode in build_segments(spans, query_offset, Sq):
|
||||
q_s = s - query_offset
|
||||
q_e = e - query_offset
|
||||
out_seg = attn_func(
|
||||
query[:, q_s:q_e],
|
||||
key[:, :e],
|
||||
value[:, :e],
|
||||
causal=(mode == "causal"),
|
||||
softmax_scale=softmax_scale,
|
||||
)
|
||||
out[:, q_s:q_e] = out_seg
|
||||
return out
|
||||
outputs = []
|
||||
covered = 0
|
||||
for local_start, local_end, global_start in ranges:
|
||||
query_len = local_end - local_start
|
||||
if local_start != covered or query_len < 0:
|
||||
raise ValueError("query_ranges must cover local query contiguously")
|
||||
for segment in build_segments(spans, global_start, query_len):
|
||||
q_start = local_start + segment.q_start - global_start
|
||||
q_end = local_start + segment.q_end - global_start
|
||||
outputs.append(
|
||||
attn_func(
|
||||
query[:, q_start:q_end],
|
||||
key[:, : segment.kv_end],
|
||||
value[:, : segment.kv_end],
|
||||
causal=(segment.mode == "causal"),
|
||||
softmax_scale=softmax_scale,
|
||||
)
|
||||
)
|
||||
covered = local_end
|
||||
|
||||
if covered != query.shape[1]:
|
||||
raise ValueError("query_ranges must cover the full local query")
|
||||
if not outputs:
|
||||
return torch.empty_like(query)
|
||||
if len(outputs) == 1:
|
||||
return outputs[0]
|
||||
return torch.cat(outputs, dim=1)
|
||||
Reference in new issue
Block a user