[Feature] Add AllGather-KV sequence-parallel attention str… (#4968)

Signed-off-by: zhuhh97 <zhuhonghan@huawei.com>
Signed-off-by: dph97 <nenbaying@163.com>
Signed-off-by: zhuhh97 <zhh@unknown>
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
Co-authored-by: dengyunyang <584797741@qq.com>
This commit is contained in:
authored and GitHub committed 2026-07-24 14:45:26 +00:00
1 parent 8927ec057c
commit 84bbd57e4d
33 files changed
+1181 -135

No files matched your search

@@ -19,7 +19,11 @@ import pytest
import torch
import torch.nn.functional as F
import vllm_omni.diffusion.attention.backends.utils.piecewise_attn as piecewise_module
from vllm_omni.diffusion.attention.backends.abstract import QueryRange
from vllm_omni.diffusion.attention.backends.utils.piecewise_attn import (
Segment,
build_segments,
piecewise_attn,
)
@@ -71,6 +75,12 @@ BATCH_CASES = [
]
def test_build_segments_full_span_retains_global_kv_end():
segments = build_segments([(2, 7)], query_offset=4, query_len=2)
assert segments == [Segment(q_start=4, q_end=6, kv_end=7, mode="full")]
@pytest.mark.parametrize("global_spans", SPAN_CASES)
@pytest.mark.parametrize("q_range", Q_RANGE_CASES)
@pytest.mark.parametrize("batch_size", BATCH_CASES)
@@ -124,3 +134,80 @@ def test_piecewise_span_fully_before_qstart():
)
expected = _full_reference(query, key, value, global_spans, q_start, q_end, softmax_scale)
torch.testing.assert_close(got, expected, atol=1e-5, rtol=1e-5)
def test_piecewise_noncontiguous_query_ranges_match_full_mask():
torch.manual_seed(0)
B, H, D, Sk = 1, 2, 8, 8
key = torch.randn(B, Sk, H, D, device=DEVICE)
value = torch.randn(B, Sk, H, D, device=DEVICE)
query = torch.randn(B, 4, H, D, device=DEVICE)
spans = [(2, 7)]
query_ranges = (
QueryRange(0, 1, 0),
QueryRange(1, 4, 4),
)
softmax_scale = D**-0.5
got = piecewise_attn(
query,
key,
value,
full_attn_spans=[spans],
softmax_scale=softmax_scale,
attn_func=_sdpa_attn_func,
query_ranges=query_ranges,
)
expected = torch.cat(
[
_full_reference(query[:, :1], key, value, spans, 0, 1, softmax_scale),
_full_reference(query[:, 1:], key, value, spans, 4, 7, softmax_scale),
],
dim=1,
)
torch.testing.assert_close(got, expected, atol=1e-5, rtol=1e-5)
def test_piecewise_rejects_incomplete_query_ranges():
query = torch.zeros(1, 3, 1, 2)
key = torch.zeros(1, 3, 1, 2)
value = torch.zeros_like(key)
with pytest.raises(ValueError, match="cover local query contiguously"):
piecewise_attn(
query,
key,
value,
full_attn_spans=[[]],
softmax_scale=1.0,
attn_func=_sdpa_attn_func,
query_ranges=(QueryRange(1, 3, 1),),
)
def test_piecewise_combines_noncontiguous_query_ranges_with_one_cat(monkeypatch):
query = torch.randn(1, 4, 2, 8)
key = torch.randn(1, 8, 2, 8)
value = torch.randn_like(key)
cat_calls = 0
original_cat = torch.cat
def counted_cat(tensors, dim=0):
nonlocal cat_calls
cat_calls += 1
return original_cat(tensors, dim=dim)
monkeypatch.setattr(piecewise_module.torch, "cat", counted_cat)
output = piecewise_attn(
query,
key,
value,
full_attn_spans=[[(0, 8)]],
softmax_scale=1.0,
attn_func=lambda q, k, v, **kwargs: q,
query_ranges=(QueryRange(0, 1, 0), QueryRange(1, 4, 4)),
)
assert cat_calls == 1
torch.testing.assert_close(output, query)