Merge remote-tracking branch 'origin/main' into qwen-image-2.1
# Conflicts: # vllm_omni/diffusion/executor/multiproc_executor.py # vllm_omni/diffusion/offloader/sequential_backend.py
This commit is contained in:
commit
9174554e40
1332 files changed
+160051
-60504
No files matched your search
@@ -10,6 +10,7 @@ Tests cover:
|
||||
- AttentionMetadata.extra field
|
||||
"""
|
||||
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
@@ -17,7 +18,12 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm_omni.diffusion.attention.layer as layer_mod
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
import vllm_omni.diffusion.attention.selector as selector_mod
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionMetadata,
|
||||
PackedPaddingMetadata,
|
||||
QueryRange,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionBackend
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.config import (
|
||||
@@ -477,6 +483,7 @@ class TestAttentionInitUsesCurrentDiffusionConfig:
|
||||
@pytest.mark.parametrize("mindiesd_available", [False, True])
|
||||
def test_paged_npu_memory_profile_uses_sdpa_only_without_mindiesd(self, monkeypatch, mindiesd_available):
|
||||
attention = Attention.__new__(Attention)
|
||||
attention.allow_fp32_fallback = False
|
||||
attention._scheduler_paged_kv = True
|
||||
attention.paged_kv_cache_role = "primary"
|
||||
attention.backend_pref = "FLASH_ATTN"
|
||||
@@ -519,6 +526,7 @@ class TestAttentionInitUsesCurrentDiffusionConfig:
|
||||
@pytest.mark.parametrize("fa4_available", [False, True])
|
||||
def test_paged_cuda_memory_profile_uses_sdpa_without_dense_fa4(self, monkeypatch, fa4_available):
|
||||
attention = Attention.__new__(Attention)
|
||||
attention.allow_fp32_fallback = False
|
||||
attention._scheduler_paged_kv = True
|
||||
attention.paged_kv_cache_role = "primary"
|
||||
attention.backend_pref = "FLASH_ATTN"
|
||||
@@ -561,6 +569,7 @@ class TestAttentionInitUsesCurrentDiffusionConfig:
|
||||
|
||||
def test_dense_flash_import_error_is_not_reclassified(self, monkeypatch):
|
||||
attention = Attention.__new__(Attention)
|
||||
attention.allow_fp32_fallback = False
|
||||
attention._scheduler_paged_kv = False
|
||||
attention.paged_kv_cache_role = "primary"
|
||||
attention.backend_pref = "FLASH_ATTN"
|
||||
@@ -900,6 +909,7 @@ class TestAttentionInitUsesCurrentDiffusionConfig:
|
||||
return sentinel
|
||||
|
||||
fake_attention = SimpleNamespace(
|
||||
allow_fp32_fallback=False,
|
||||
attention=SimpleNamespace(
|
||||
forward=_selected_forward,
|
||||
),
|
||||
@@ -938,6 +948,164 @@ class TestAttentionInitUsesCurrentDiffusionConfig:
|
||||
Attention._run_ring_attention(fake_attention, query, query, query, metadata)
|
||||
|
||||
|
||||
class TestOptInFloat32Fallback:
|
||||
@staticmethod
|
||||
def _make_attention(monkeypatch, *, backend=None, allow=True, kv_cache_dtype=None):
|
||||
events, state = [], {}
|
||||
context = object()
|
||||
|
||||
class _SelectedImpl:
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
|
||||
@staticmethod
|
||||
def supports_kv_cache_dtype(*args):
|
||||
return True
|
||||
|
||||
def forward(self, query, key, value, metadata):
|
||||
events.append("selected")
|
||||
state["kernel_call"] = (query, key, value, metadata)
|
||||
state["kernel_output"] = query + 4
|
||||
return state["kernel_output"]
|
||||
|
||||
class _FallbackImpl(_SelectedImpl):
|
||||
def forward(self, query, key, value, metadata):
|
||||
output = super().forward(query, key, value, metadata)
|
||||
events[-1] = "fallback"
|
||||
return output
|
||||
|
||||
class _FlashBackend(FlashAttentionBackend):
|
||||
@staticmethod
|
||||
def get_impl_cls():
|
||||
return _SelectedImpl
|
||||
|
||||
class _SDPABackend(layer_mod.SDPABackend):
|
||||
@staticmethod
|
||||
def get_impl_cls():
|
||||
return _SelectedImpl
|
||||
|
||||
class _Strategy:
|
||||
def pre_attention(self, query, key, value, metadata):
|
||||
events.append("pre")
|
||||
state["prepared"] = (
|
||||
query + 1,
|
||||
key + 2,
|
||||
value + 3,
|
||||
replace(metadata) if metadata is not None else None,
|
||||
)
|
||||
return (*state["prepared"], context)
|
||||
|
||||
def post_attention(self, output, ctx):
|
||||
events.append("post")
|
||||
assert output is state["kernel_output"]
|
||||
assert ctx is context
|
||||
return output + 7
|
||||
|
||||
# Keep real role/config resolution and construction; replace hardware
|
||||
# lookup and kernels. These are CPU dispatch tests, not SP collectives.
|
||||
monkeypatch.setattr(
|
||||
selector_mod,
|
||||
"_cached_get_backend_cls",
|
||||
lambda name, *args, **kwargs: _SDPABackend if name == "TORCH_SDPA" else _FlashBackend,
|
||||
)
|
||||
monkeypatch.setattr(layer_mod.SDPABackend, "get_impl_cls", staticmethod(lambda: _FallbackImpl))
|
||||
monkeypatch.setattr(layer_mod, "build_parallel_attention_strategy", lambda **kwargs: _Strategy())
|
||||
monkeypatch.setattr(layer_mod, "is_forward_context_available", lambda: False)
|
||||
config = OmniDiffusionConfig(
|
||||
diffusion_attention_config={"default": backend} if backend else {},
|
||||
diffusion_kv_cache_dtype=kv_cache_dtype,
|
||||
)
|
||||
with set_current_diffusion_config(config):
|
||||
attention = Attention(
|
||||
num_heads=4,
|
||||
num_kv_heads=2,
|
||||
head_size=8,
|
||||
causal=False,
|
||||
softmax_scale=0.25,
|
||||
**({"allow_fp32_fallback": True} if allow else {}),
|
||||
)
|
||||
return attention, events, state
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend,allow,dtype,is_cuda,expected",
|
||||
[
|
||||
pytest.param(None, True, torch.float32, True, "fallback", id="auto-fp32"),
|
||||
pytest.param(None, False, torch.float32, True, "selected", id="default-opt-out"),
|
||||
pytest.param("FLASH_ATTN", True, torch.float32, True, "selected", id="explicit-flash"),
|
||||
pytest.param("TORCH_SDPA", True, torch.float32, True, "selected", id="explicit-sdpa"),
|
||||
pytest.param(None, True, torch.bfloat16, True, "selected", id="bf16"),
|
||||
pytest.param(None, True, torch.float32, False, "selected", id="cpu-fp32"),
|
||||
],
|
||||
)
|
||||
def test_public_forward_preserves_strategy_and_backend_choice(
|
||||
self, monkeypatch, backend, allow, dtype, is_cuda, expected
|
||||
):
|
||||
attention, events, state = self._make_attention(monkeypatch, backend=backend, allow=allow)
|
||||
# Only the dispatch predicate is simulated; every tensor stays on CPU.
|
||||
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: is_cuda))
|
||||
query = torch.ones(1, 2, 4, 8, dtype=dtype)
|
||||
key = torch.full((1, 2, 2, 8), 2, dtype=dtype)
|
||||
value = torch.full_like(key, 3)
|
||||
metadata = AttentionMetadata(attn_mask=torch.tensor([[True, False]]))
|
||||
|
||||
output = attention(query, key, value, metadata)
|
||||
|
||||
assert events == ["pre", expected, "post"]
|
||||
assert all(got is prepared for got, prepared in zip(state["kernel_call"], state["prepared"]))
|
||||
assert state["kernel_call"][3] is not metadata
|
||||
assert state["kernel_call"][3].attn_mask is metadata.attn_mask
|
||||
torch.testing.assert_close(output, query + 12)
|
||||
assert attention.backend_explicit is (backend is not None)
|
||||
impl = attention.sdpa_fallback if expected == "fallback" else attention.attention
|
||||
assert impl.kwargs["num_kv_heads"] == 2
|
||||
assert impl.kwargs["softmax_scale"] == 0.25
|
||||
|
||||
def test_explicit_backend_failure_is_not_retried_with_sdpa(self, monkeypatch):
|
||||
attention, events, _ = self._make_attention(monkeypatch, backend="FLASH_ATTN")
|
||||
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
|
||||
selected = Mock(side_effect=RuntimeError("selected backend rejects this dtype"))
|
||||
monkeypatch.setattr(attention.attention, "forward", selected)
|
||||
query = torch.ones(1, 2, 4, 8)
|
||||
|
||||
with pytest.raises(RuntimeError, match="selected backend rejects this dtype"):
|
||||
attention(query, query, query)
|
||||
|
||||
selected.assert_called_once()
|
||||
assert events == ["pre"]
|
||||
|
||||
@pytest.mark.parametrize("contract", ["packed", "packed-padding", "piecewise", "quantized-kv"])
|
||||
def test_backend_metadata_is_not_silently_sent_to_dense_sdpa(self, monkeypatch, contract):
|
||||
quant_dtype = "fp8" if contract == "quantized-kv" else None
|
||||
attention, events, state = self._make_attention(monkeypatch, kv_cache_dtype=quant_dtype)
|
||||
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
|
||||
lengths = torch.tensor([0, 1, 2], dtype=torch.int32)
|
||||
metadata = AttentionMetadata()
|
||||
if contract == "packed":
|
||||
metadata.extra = {
|
||||
"cu_seqlens_q": lengths,
|
||||
"cu_seqlens_k": lengths,
|
||||
"max_seqlen_q": 1,
|
||||
"max_seqlen_k": 1,
|
||||
}
|
||||
elif contract == "packed-padding":
|
||||
metadata.packed_padding = PackedPaddingMetadata(1, 1, lengths[:2], lengths[:2])
|
||||
elif contract == "piecewise":
|
||||
metadata.full_attn_spans = [[(0, 1)]]
|
||||
metadata.query_ranges = (QueryRange(0, 2, 0),)
|
||||
query = torch.ones(1, 2, 4, 8)
|
||||
|
||||
attention(query, query, query, metadata)
|
||||
|
||||
assert events == ["pre", "selected", "post"]
|
||||
forwarded = state["kernel_call"][3]
|
||||
if quant_dtype:
|
||||
# Quantization is injected by the public forward from config.
|
||||
assert forwarded.extra == {"kv_cache_dtype": "fp8"}
|
||||
assert metadata.extra == {}
|
||||
else:
|
||||
assert forwarded is state["prepared"][3]
|
||||
|
||||
|
||||
class TestDiffusionKvCacheQuantization:
|
||||
@staticmethod
|
||||
def _install_attention_init_stubs(monkeypatch, *, supports_kv_cache_dtype=True):
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import sys
|
||||
import types
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
|
||||
from vllm_omni.diffusion.attention.ops.block_sparse import (
|
||||
build_prefix_dense_block_map,
|
||||
fastvideo_block_sparse_attn_bshd,
|
||||
mean_pool_tiles,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prefix_blocks,topk", [(0, 1), (2, 0), (2, 1), (2, 3), (2, 5)])
|
||||
def test_block_map_keeps_prefix_dense_and_selects_non_prefix_keys(prefix_blocks, topk):
|
||||
sparse_blocks = 3
|
||||
blocks = prefix_blocks + sparse_blocks
|
||||
scores = torch.arange(2 * blocks * blocks, dtype=torch.float32).reshape(1, 2, blocks, blocks)
|
||||
block_map = build_prefix_dense_block_map(scores, prefix_blocks, sparse_blocks, topk)
|
||||
expected = torch.zeros_like(scores, dtype=torch.bool)
|
||||
# Scores increase with key index, so top-k always selects the last keys.
|
||||
for query_block in range(blocks):
|
||||
for key_block in range(blocks):
|
||||
if (
|
||||
query_block < prefix_blocks
|
||||
or key_block < prefix_blocks
|
||||
or key_block >= blocks - min(topk, sparse_blocks)
|
||||
):
|
||||
expected[..., query_block, key_block] = True
|
||||
assert torch.equal(block_map, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_size", [4, 32, 64])
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
def test_pooling_uses_valid_tile_sizes_and_fp32_accumulation(block_size, dtype):
|
||||
sizes = torch.tensor([block_size, block_size - 1, 0], dtype=torch.int32)
|
||||
tiles = torch.arange(2 * 3 * block_size * 2 * 3).reshape(2, 3, block_size, 2, 3).to(dtype)
|
||||
tiles[:, 1, -1] = 0
|
||||
tiles[:, 2] = 0
|
||||
expected = torch.stack(
|
||||
[tiles[:, 0].float().mean(dim=1), tiles[:, 1, :-1].float().mean(dim=1), torch.zeros(2, 2, 3)], dim=2
|
||||
)
|
||||
result = mean_pool_tiles(tiles.flatten(1, 2), sizes, block_size)
|
||||
assert result.dtype == torch.float32
|
||||
torch.testing.assert_close(result, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_provider_wrapper_restores_transport_rows_and_bshd_layout(monkeypatch):
|
||||
monkeypatch.delenv("FASTVIDEO_VSA_SM100A", raising=False)
|
||||
calls = []
|
||||
|
||||
def block_sparse_attn(q, k, v, block_map, sizes):
|
||||
calls.append((q.shape, block_map.shape, sizes.tolist()))
|
||||
assert q.is_contiguous() and k.is_contiguous() and v.is_contiguous()
|
||||
assert sizes.dtype == torch.int32
|
||||
return q + k + v, None
|
||||
|
||||
module = types.ModuleType("fastvideo_kernel.block_sparse_attn")
|
||||
setattr(module, "block_sparse_attn", block_sparse_attn)
|
||||
monkeypatch.setitem(sys.modules, "fastvideo_kernel", types.ModuleType("fastvideo_kernel"))
|
||||
monkeypatch.setitem(sys.modules, "fastvideo_kernel.block_sparse_attn", module)
|
||||
query = torch.randn(1, 4 * 64, 2, 8)
|
||||
block_map = torch.ones(1, 2, 4, 4, dtype=torch.bool)
|
||||
sizes = torch.tensor([64, 17, 64, 0])
|
||||
output = fastvideo_block_sparse_attn_bshd(query, query, query, block_map, sizes, 3)
|
||||
expected = torch.zeros_like(query)
|
||||
expected[:, : 3 * 64] = query[:, : 3 * 64] * 3
|
||||
torch.testing.assert_close(output, expected)
|
||||
assert calls == [(torch.Size([1, 2, 3 * 64, 8]), torch.Size([1, 2, 3, 3]), [64, 17, 64])]
|
||||
|
||||
|
||||
def test_custom_op_fake_preserves_metadata_without_loading_provider(monkeypatch):
|
||||
monkeypatch.setitem(sys.modules, "fastvideo_kernel", None)
|
||||
with FakeTensorMode():
|
||||
query = torch.empty(1, 4 * 64, 2, 8, dtype=torch.bfloat16)
|
||||
block_map = torch.empty(1, 2, 4, 4, dtype=torch.bool)
|
||||
sizes = torch.empty(4, dtype=torch.int32)
|
||||
output = fastvideo_block_sparse_attn_bshd(query, query, query, block_map, sizes, 3)
|
||||
assert output.shape == query.shape
|
||||
assert output.dtype == query.dtype
|
||||
assert output.device == query.device
|
||||
|
||||
|
||||
@pytest.mark.parametrize("error_type", [ImportError, RuntimeError, torch.AcceleratorError])
|
||||
def test_native_provider_failure_only_recovers_without_accelerator_fault(monkeypatch, mocker, error_type):
|
||||
monkeypatch.setenv("FASTVIDEO_VSA_SM100A", "1")
|
||||
error = error_type("native provider failure")
|
||||
native = mocker.Mock()
|
||||
native.is_supported.return_value = True
|
||||
native.block_sparse_attn_sm100a.side_effect = error
|
||||
provider = types.ModuleType("fastvideo_kernel")
|
||||
setattr(provider, "block_sparse_attn_sm100a", native)
|
||||
index = types.ModuleType("fastvideo_kernel.triton_kernels.index")
|
||||
setattr(
|
||||
index,
|
||||
"map_to_index",
|
||||
mocker.Mock(return_value=(torch.zeros(1, dtype=torch.int32), torch.ones(1, dtype=torch.int32))),
|
||||
)
|
||||
triton = types.ModuleType("fastvideo_kernel.block_sparse_attn")
|
||||
fallback = mocker.Mock(side_effect=lambda q, *args: (q.clone(), None))
|
||||
setattr(triton, "block_sparse_attn", fallback)
|
||||
monkeypatch.setitem(sys.modules, "fastvideo_kernel", provider)
|
||||
monkeypatch.setitem(
|
||||
sys.modules, "fastvideo_kernel.triton_kernels", types.ModuleType("fastvideo_kernel.triton_kernels")
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "fastvideo_kernel.triton_kernels.index", index)
|
||||
monkeypatch.setitem(sys.modules, "fastvideo_kernel.block_sparse_attn", triton)
|
||||
query = torch.arange(128 * 2 * 8, dtype=torch.float32).reshape(1, 128, 2, 8)
|
||||
block_map = torch.ones(1, 2, 2, 2, dtype=torch.bool)
|
||||
sizes = torch.tensor([64, 64], dtype=torch.int32)
|
||||
|
||||
if error_type is torch.AcceleratorError:
|
||||
with pytest.raises(torch.AcceleratorError, match="native provider failure") as raised:
|
||||
fastvideo_block_sparse_attn_bshd(query, query, query, block_map, sizes, 2)
|
||||
assert raised.value is error
|
||||
fallback.assert_not_called()
|
||||
else:
|
||||
output = fastvideo_block_sparse_attn_bshd(query, query, query, block_map, sizes, 2)
|
||||
torch.testing.assert_close(output, query, rtol=0, atol=0)
|
||||
fallback.assert_called_once()
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import math
|
||||
import sys
|
||||
import types
|
||||
from typing import Any
|
||||
@@ -12,14 +11,10 @@ import torch
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionMetadata,
|
||||
PackedPaddingMetadata,
|
||||
VideoTokenLayout,
|
||||
VideoTokenSpan,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.backends.fastvideo_vsa import (
|
||||
FastVideoVSABackend,
|
||||
FastVideoVSAImpl,
|
||||
_build_h3_block_map,
|
||||
_get_h3_tile_metadata,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.backends.registry import (
|
||||
DiffusionAttentionBackendEnum,
|
||||
@@ -218,144 +213,8 @@ def test_fastvideo_vsa_allows_topk_equal_to_num_blocks():
|
||||
assert too_many._fallback_reason(query, query, query, metadata) == "topk 3 > num_blocks 2"
|
||||
|
||||
|
||||
def _h3_metadata(prefix_segments, video_shape, *, gate=None, packed_padding=None):
|
||||
extra: dict[str, Any] = {
|
||||
"vsa_h3_prefix_segments": prefix_segments,
|
||||
}
|
||||
if gate is not None:
|
||||
extra["gate_compress"] = gate
|
||||
return AttentionMetadata(
|
||||
packed_padding=packed_padding,
|
||||
video_layout=VideoTokenLayout(
|
||||
used_len=sum(prefix_segments) + math.prod(video_shape),
|
||||
video_spans=(VideoTokenSpan(start=sum(prefix_segments), latent_grid=video_shape, role="target"),),
|
||||
),
|
||||
extra=extra,
|
||||
)
|
||||
|
||||
|
||||
def _fake_block_sparse_kernel(monkeypatch, calls):
|
||||
"""Stand in for fastvideo_kernel.block_sparse_attn with a mask-obeying SDPA."""
|
||||
|
||||
def block_sparse_attn(q, k, v, block_map, variable_block_sizes):
|
||||
calls["block_map"] = block_map.clone()
|
||||
calls["variable_block_sizes"] = variable_block_sizes.tolist()
|
||||
calls["q_shape"] = tuple(q.shape)
|
||||
blocks = block_map.shape[-1]
|
||||
assert blocks * 64 == q.shape[2]
|
||||
# Expand the per-block map to a per-token mask so the stand-in honours
|
||||
# the sparsity contract the real kernel implements, and drop the rows
|
||||
# each partial block pads with, which the real kernel skips through
|
||||
# variable_block_sizes.
|
||||
token_mask = block_map.repeat_interleave(64, dim=-1).repeat_interleave(64, dim=-2)
|
||||
within = torch.arange(64, device=q.device)[None, :] < variable_block_sizes[:, None]
|
||||
token_mask = token_mask & within.reshape(1, 1, 1, -1)
|
||||
out = torch.nn.functional.scaled_dot_product_attention(q.float(), k.float(), v.float(), attn_mask=token_mask)
|
||||
return out.to(q.dtype), None
|
||||
|
||||
module = types.ModuleType("fastvideo_kernel.block_sparse_attn")
|
||||
setattr(module, "block_sparse_attn", block_sparse_attn)
|
||||
monkeypatch.setitem(sys.modules, "fastvideo_kernel", types.ModuleType("fastvideo_kernel"))
|
||||
monkeypatch.setitem(sys.modules, "fastvideo_kernel.block_sparse_attn", module)
|
||||
|
||||
|
||||
def _h3_impl(**backend_kwargs):
|
||||
kwargs = {"topk": 1, "min_seq_len": 1, "disable_when_sp_active": False}
|
||||
kwargs.update(backend_kwargs)
|
||||
return FastVideoVSAImpl(num_heads=2, head_size=8, softmax_scale=8**-0.5, backend_kwargs=kwargs)
|
||||
|
||||
|
||||
def test_h3_forward_tiles_untiles_and_restores_the_original_row_order(monkeypatch):
|
||||
calls: dict[str, Any] = {}
|
||||
_fake_block_sparse_kernel(monkeypatch, calls)
|
||||
impl = _h3_impl(topk=64)
|
||||
prefix_segments, video_shape = (5, 70), (2, 4, 4)
|
||||
seq_len = sum(prefix_segments) + 2 * 4 * 4
|
||||
torch.manual_seed(0)
|
||||
query = torch.randn(1, seq_len, 2, 8, dtype=torch.bfloat16)
|
||||
|
||||
output = impl.forward_cuda(query, query, query, _h3_metadata(prefix_segments, video_shape))
|
||||
|
||||
assert output.shape == query.shape
|
||||
# topk covers every video block, so the block map is dense and the tiled
|
||||
# kernel must reproduce plain attention over the same rows.
|
||||
reference = torch.nn.functional.scaled_dot_product_attention(
|
||||
*(x.float().transpose(1, 2) for x in (query, query, query))
|
||||
).transpose(1, 2)
|
||||
torch.testing.assert_close(output.float(), reference, atol=2e-2, rtol=2e-2)
|
||||
# 5 + 70 splits into segment-pure 5 | 64 | 6, then one (2, 4, 4) video tile.
|
||||
assert calls["variable_block_sizes"] == [5, 64, 6, 32]
|
||||
|
||||
|
||||
def test_h3_forward_keeps_prefix_dense_and_selects_top_k_video_tiles(monkeypatch):
|
||||
calls: dict[str, Any] = {}
|
||||
_fake_block_sparse_kernel(monkeypatch, calls)
|
||||
impl = _h3_impl(topk=1)
|
||||
prefix_segments, video_shape = (5,), (8, 4, 4)
|
||||
seq_len = 5 + 8 * 4 * 4
|
||||
torch.manual_seed(0)
|
||||
query = torch.randn(1, seq_len, 2, 8, dtype=torch.bfloat16)
|
||||
|
||||
impl.forward_cuda(query, query, query, _h3_metadata(prefix_segments, video_shape))
|
||||
|
||||
# 1 prefix block + 2 video blocks. The odd count is padded to an even one
|
||||
# for the paired-CTA kernel contract, and the Triton route slices that
|
||||
# transport-only partner off again before launching.
|
||||
block_map = calls["block_map"]
|
||||
assert block_map.shape[-1] == 3
|
||||
assert calls["variable_block_sizes"] == [5, 64, 64]
|
||||
assert block_map[:, :, :1, :].all(), "prefix queries stay dense"
|
||||
assert block_map[..., :1].all(), "prefix keys are exempt from selection"
|
||||
assert (block_map[:, :, 1:, 1:].sum(dim=-1) == 1).all(), "one video tile per video query"
|
||||
|
||||
# The same odd geometry must still untile back onto the original rows.
|
||||
dense = _h3_impl(topk=64).forward_cuda(query, query, query, _h3_metadata(prefix_segments, video_shape))
|
||||
reference = torch.nn.functional.scaled_dot_product_attention(
|
||||
*(x.float().transpose(1, 2) for x in (query, query, query))
|
||||
).transpose(1, 2)
|
||||
torch.testing.assert_close(dense.float(), reference, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
def test_h3_forward_applies_the_learned_compression_gate(monkeypatch):
|
||||
_fake_block_sparse_kernel(monkeypatch, {})
|
||||
impl = _h3_impl(topk=64)
|
||||
prefix_segments, video_shape = (5,), (2, 4, 4)
|
||||
seq_len = 5 + 2 * 4 * 4
|
||||
torch.manual_seed(0)
|
||||
query = torch.randn(1, seq_len, 2, 8, dtype=torch.bfloat16)
|
||||
|
||||
without_gate = impl.forward_cuda(query, query, query, _h3_metadata(prefix_segments, video_shape))
|
||||
zero_gate = torch.zeros_like(query)
|
||||
with_zero_gate = impl.forward_cuda(query, query, query, _h3_metadata(prefix_segments, video_shape, gate=zero_gate))
|
||||
live_gate = torch.full_like(query, 0.5)
|
||||
with_live_gate = impl.forward_cuda(query, query, query, _h3_metadata(prefix_segments, video_shape, gate=live_gate))
|
||||
|
||||
torch.testing.assert_close(with_zero_gate.float(), without_gate.float())
|
||||
assert not torch.allclose(with_live_gate.float(), without_gate.float())
|
||||
|
||||
|
||||
def test_h3_forward_restores_rows_the_packed_padding_excludes(monkeypatch):
|
||||
_fake_block_sparse_kernel(monkeypatch, {})
|
||||
impl = _h3_impl(topk=64)
|
||||
prefix_segments, video_shape = (5,), (2, 4, 4)
|
||||
valid = 5 + 2 * 4 * 4
|
||||
torch.manual_seed(0)
|
||||
query = torch.randn(1, valid + 24, 2, 8, dtype=torch.bfloat16)
|
||||
padding = PackedPaddingMetadata(
|
||||
q_length=valid,
|
||||
kv_length=valid,
|
||||
cu_seqlens_q=torch.tensor([0, valid], dtype=torch.int32),
|
||||
cu_seqlens_k=torch.tensor([0, valid], dtype=torch.int32),
|
||||
)
|
||||
|
||||
output = impl.forward_cuda(query, query, query, _h3_metadata(prefix_segments, video_shape, packed_padding=padding))
|
||||
|
||||
assert output.shape == query.shape
|
||||
assert torch.count_nonzero(output[:, valid:]) == 0
|
||||
|
||||
|
||||
def test_sdpa_fallback_never_attends_the_structural_padding(monkeypatch):
|
||||
# The backend advertises supports_packed_mask_free, so MiniMax-H3 skips
|
||||
# The backend advertises supports_packed_mask_free, so a model may skip
|
||||
# building the padding mask. Every fallback out of the VSA path must honour
|
||||
# that contract itself; SDPA reads attn_mask and nothing else.
|
||||
impl = FastVideoVSAImpl(
|
||||
@@ -374,6 +233,8 @@ def test_sdpa_fallback_never_attends_the_structural_padding(monkeypatch):
|
||||
cu_seqlens_k=torch.tensor([0, valid], dtype=torch.int32),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(impl.sdpa_fallback, "forward", impl.sdpa_fallback.forward_cuda)
|
||||
|
||||
baseline = impl.forward_cuda(query, query, query, AttentionMetadata(packed_padding=padding, extra={}))
|
||||
perturbed_input = query.clone()
|
||||
perturbed_input[:, valid:] += 100.0
|
||||
@@ -384,22 +245,3 @@ def test_sdpa_fallback_never_attends_the_structural_padding(monkeypatch):
|
||||
assert baseline.shape == query.shape
|
||||
torch.testing.assert_close(baseline[:, :valid], perturbed[:, :valid])
|
||||
assert torch.count_nonzero(baseline[:, valid:]) == 0
|
||||
|
||||
|
||||
def test_h3_geometry_keeps_prefix_segments_pure_and_tiles_video_3d():
|
||||
partition, sizes, non_pad, untile, prefix_blocks, video_blocks = _get_h3_tile_metadata(
|
||||
(5, 70, 9), (5, 6, 6), torch.device("cpu")
|
||||
)
|
||||
assert sizes[:4].tolist() == [5, 64, 6, 9]
|
||||
assert prefix_blocks == 4
|
||||
assert video_blocks == 8
|
||||
assert int(sizes.sum()) == 5 + 70 + 9 + 5 * 6 * 6
|
||||
assert partition.numel() == non_pad.numel() == untile.numel() == int(sizes.sum())
|
||||
|
||||
|
||||
def test_h3_block_map_makes_prefix_queries_dense_and_prefix_keys_exempt():
|
||||
scores = torch.arange(1 * 2 * 5 * 5, dtype=torch.float32).reshape(1, 2, 5, 5)
|
||||
block_map = _build_h3_block_map(scores, num_prefix_blocks=2, num_video_blocks=3, topk=1)
|
||||
assert block_map[:, :, :2].all()
|
||||
assert block_map[..., :2].all()
|
||||
assert (block_map[:, :, 2:, 2:].sum(dim=-1) == 1).all()
|
||||
@@ -16,13 +16,14 @@ from unittest.mock import Mock
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionImpl
|
||||
from vllm_omni.diffusion.attention.backends.sdpa import SDPAImpl
|
||||
from vllm_omni.diffusion.attention.backends.utils import fa # noqa: E402
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
|
||||
|
||||
is_gpu = current_omni_platform.is_cuda_alike() or current_omni_platform.is_xpu()
|
||||
HAS_FLASH_ATTN = fa.HAS_FLASH_ATTN
|
||||
@@ -42,10 +43,12 @@ flash_attn_func = fa.flash_attn_func # noqa: N813
|
||||
(9, "4", frozenset((2, 3, 4)), 4),
|
||||
],
|
||||
)
|
||||
@pytest.mark.cpu
|
||||
def test_versioned_flash_attention_selection(device_major, requested, supported, expected):
|
||||
assert fa._choose_vllm_flash_attn_version(device_major, requested, supported) == expected
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_versioned_flash_attention_selection_rejects_invalid_or_unavailable_version():
|
||||
with pytest.raises(ValueError, match="must be 2, 3, or 4"):
|
||||
fa._choose_vllm_flash_attn_version(9, "5", frozenset((2, 3, 4)))
|
||||
@@ -92,7 +95,8 @@ def pad_tensor(tensor: torch.Tensor, target_seq_len: int, pad_value: float = 0.0
|
||||
return torch.cat([tensor, padding], dim=1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
|
||||
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires an accelerator")
|
||||
def test_padding_equivalence():
|
||||
"""
|
||||
Case 1: Test that padded and unpadded inputs produce similar outputs.
|
||||
@@ -185,7 +189,8 @@ def test_padding_equivalence():
|
||||
print("✓ Case 1 PASSED: Padded and unpadded outputs are very close!")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
|
||||
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires an accelerator")
|
||||
def test_fa_vs_sdpa():
|
||||
"""
|
||||
Case 2: Compare FlashAttention and SDPA backends with padding.
|
||||
@@ -302,7 +307,8 @@ def test_fa_vs_sdpa():
|
||||
print("✓ Case 2 PASSED: FA and SDPA outputs are very close!")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
|
||||
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires an accelerator")
|
||||
def test_flash_attn_func_preferred_over_varlen():
|
||||
"""Test flash_attn_func availability and basic forward call."""
|
||||
if not HAS_FLASH_ATTN:
|
||||
@@ -332,7 +338,8 @@ def test_flash_attn_func_preferred_over_varlen():
|
||||
print("✓ flash_attn_func forward works correctly!")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
|
||||
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires an accelerator")
|
||||
@pytest.mark.parametrize("k_len", [40, 256])
|
||||
def test_cross_attn_key_padding_vs_sdpa(k_len):
|
||||
"""
|
||||
@@ -390,6 +397,7 @@ def test_cross_attn_key_padding_vs_sdpa(k_len):
|
||||
print("✓ Case 3 PASSED: FA and SDPA cross-attention outputs are very close!")
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_varlen_masked_routing_by_role(monkeypatch):
|
||||
"""The unpad route is picked by role, not Q/K length equality; runs without a GPU."""
|
||||
calls = []
|
||||
@@ -424,6 +432,7 @@ def test_varlen_masked_routing_by_role(monkeypatch):
|
||||
assert cu_seqlens_k.tolist() == [0, 2, 5]
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_piecewise_flash_attn_uses_varlen_fallback(monkeypatch):
|
||||
calls = []
|
||||
|
||||
@@ -454,6 +463,7 @@ def test_piecewise_flash_attn_uses_varlen_fallback(monkeypatch):
|
||||
assert calls[0]["softmax_scale"] == 0.5
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_packed_varlen_metadata_bypasses_mask_unpadding(monkeypatch):
|
||||
calls = []
|
||||
|
||||
@@ -486,6 +496,7 @@ def test_packed_varlen_metadata_bypasses_mask_unpadding(monkeypatch):
|
||||
assert calls[0][3]["max_seqlen_q"] == 6
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_packed_varlen_metadata_must_be_complete(monkeypatch):
|
||||
monkeypatch.setattr(fa, "HAS_FLASH_ATTN", True)
|
||||
impl = FlashAttentionImpl(num_heads=2, head_size=4, softmax_scale=0.5, causal=False)
|
||||
@@ -503,11 +514,12 @@ def test_packed_varlen_metadata_must_be_complete(monkeypatch):
|
||||
# - boundary resolution in ``_resolve_packed_seq_npu`` (pure Python, no device
|
||||
# sync, no MindIE-SD import);
|
||||
# - env dispatch in ``forward_fa_npu`` (which op a packed forward takes);
|
||||
# - native torch_npu dispatch for right-down causal attention;
|
||||
# - the laser input pre-scaling math in ``_forward_prefix_kv_slice_npu``.
|
||||
#
|
||||
# They run on CPU without a real NPU by injecting a fake ``mindiesd`` module
|
||||
# into ``sys.modules`` (same pattern as ``test_paged_attention`` /
|
||||
# ``benchmarks/conftest``). No production code is exercised for real kernels.
|
||||
# They run on CPU without a real NPU by injecting fake ``mindiesd`` and
|
||||
# ``torch_npu`` modules into ``sys.modules`` (same pattern as
|
||||
# ``test_paged_attention`` / ``benchmarks/conftest``). No real kernel runs.
|
||||
|
||||
|
||||
def _npu_impl(causal: bool = False) -> FlashAttentionImpl:
|
||||
@@ -529,9 +541,19 @@ def _fake_mindiesd(monkeypatch, *, attention_forward=None, attention_forward_var
|
||||
return fake
|
||||
|
||||
|
||||
def _fake_torch_npu(monkeypatch, *, npu_fusion_attention=None):
|
||||
"""Install the minimal torch_npu surface used by causal NPU attention."""
|
||||
fake = SimpleNamespace(
|
||||
npu_fusion_attention=npu_fusion_attention or Mock(return_value=(torch.zeros(1), None, None, None, 0, 0, 0)),
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "torch_npu", fake)
|
||||
return fake
|
||||
|
||||
|
||||
# --- Test group A: boundary resolution (_resolve_packed_seq_npu) -------------
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_resolve_packed_seq_accepts_real_plus_pad_contract():
|
||||
impl = _npu_impl()
|
||||
q = torch.randn(1, 8, 2, 4)
|
||||
@@ -541,6 +563,7 @@ def test_resolve_packed_seq_accepts_real_plus_pad_contract():
|
||||
assert impl._resolve_packed_seq_npu(q, q, extra) == ([5, 8], [5, 8])
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_resolve_packed_seq_single_document_no_padding():
|
||||
impl = _npu_impl()
|
||||
q = torch.randn(1, 8, 2, 4)
|
||||
@@ -564,6 +587,7 @@ def test_resolve_packed_seq_single_document_no_padding():
|
||||
"pad_longer_than_real",
|
||||
],
|
||||
)
|
||||
@pytest.mark.cpu
|
||||
def test_resolve_packed_seq_rejects_malformed_metadata(case):
|
||||
"""Any metadata outside the exact [real, pad] contract returns None so the
|
||||
caller falls back to the masked path."""
|
||||
@@ -620,6 +644,7 @@ def _npu_impl_with_mocked_paths() -> tuple[FlashAttentionImpl, torch.Tensor]:
|
||||
return impl, sentinel
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_npu_env_unset_uses_varlen(monkeypatch):
|
||||
monkeypatch.delenv("MINDIE_SD_FA_TYPE", raising=False)
|
||||
_fake_mindiesd(monkeypatch)
|
||||
@@ -632,6 +657,7 @@ def test_npu_env_unset_uses_varlen(monkeypatch):
|
||||
impl._forward_prefix_kv_slice_npu.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_npu_env_laser_uses_slice(monkeypatch):
|
||||
monkeypatch.setenv("MINDIE_SD_FA_TYPE", "ascend_laser_attention")
|
||||
_fake_mindiesd(monkeypatch)
|
||||
@@ -645,6 +671,7 @@ def test_npu_env_laser_uses_slice(monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fa_type", ["prompt_flash_attn", "fused_attn_score", "ascend_laser_attenton", "garbage"])
|
||||
@pytest.mark.cpu
|
||||
def test_npu_env_non_laser_value_falls_back_to_varlen(monkeypatch, fa_type):
|
||||
"""Locks in the CURRENT dispatch behavior (PR #5891).
|
||||
|
||||
@@ -665,6 +692,7 @@ def test_npu_env_non_laser_value_falls_back_to_varlen(monkeypatch, fa_type):
|
||||
impl._forward_prefix_kv_slice_npu.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_npu_varlen_opt_in_unset_takes_mask_path(monkeypatch):
|
||||
"""Models that do not set ``npu_attn_varlen`` (e.g. Wan/Cosmos) keep using
|
||||
the existing masked attention_forward path; the new branches are skipped."""
|
||||
@@ -685,9 +713,115 @@ def test_npu_varlen_opt_in_unset_takes_mask_path(monkeypatch):
|
||||
assert out is fake_forward.return_value
|
||||
|
||||
|
||||
# --- Test group C: laser input pre-scaling in _forward_prefix_kv_slice_npu --
|
||||
# --- Test group C: native NPU causal attention dispatch ----------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query_length", "key_length", "expected_inner_precise"),
|
||||
[(4, 4, 0), (2, 4, 0), (4, 2, 2)],
|
||||
)
|
||||
@pytest.mark.cpu
|
||||
def test_npu_causal_uses_native_right_down_mode(
|
||||
monkeypatch,
|
||||
query_length,
|
||||
key_length,
|
||||
expected_inner_precise,
|
||||
):
|
||||
expected_out = torch.randn(1, query_length, 2, 4)
|
||||
fusion_attention = Mock(return_value=(expected_out, None, None, None, 0, 0, 0))
|
||||
fake_torch_npu = _fake_torch_npu(monkeypatch, npu_fusion_attention=fusion_attention)
|
||||
impl = FlashAttentionImpl(num_heads=8, head_size=4, softmax_scale=0.5, causal=True)
|
||||
query = torch.randn(1, query_length, 2, 4)
|
||||
key = torch.randn(1, key_length, 2, 4)
|
||||
|
||||
out = impl.forward_fa_npu(query, key, key)
|
||||
|
||||
fake_torch_npu.npu_fusion_attention.assert_called_once()
|
||||
args, kwargs = fake_torch_npu.npu_fusion_attention.call_args
|
||||
assert torch.equal(args[0], query)
|
||||
assert torch.equal(args[1], key)
|
||||
assert torch.equal(args[2], key)
|
||||
assert all(tensor.is_contiguous() for tensor in args[:3])
|
||||
assert kwargs["head_num"] == query.shape[2]
|
||||
assert kwargs["input_layout"] == "BSND"
|
||||
assert kwargs["scale"] == 0.5
|
||||
assert kwargs["keep_prob"] == 1.0
|
||||
assert kwargs["sparse_mode"] == 3
|
||||
assert kwargs["inner_precise"] == expected_inner_precise
|
||||
|
||||
block_mask = kwargs["atten_mask"]
|
||||
assert block_mask.shape == (2048, 2048)
|
||||
assert block_mask.dtype == torch.bool
|
||||
assert block_mask.is_contiguous()
|
||||
expected_corner = torch.tensor(
|
||||
[
|
||||
[False, True, True, True],
|
||||
[False, False, True, True],
|
||||
[False, False, False, True],
|
||||
[False, False, False, False],
|
||||
]
|
||||
)
|
||||
assert torch.equal(block_mask[:4, :4], expected_corner)
|
||||
assert out is expected_out
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_npu_causal_composes_explicit_keep_mask(monkeypatch):
|
||||
expected_out = torch.randn(1, 4, 2, 4)
|
||||
fusion_attention = Mock(return_value=(expected_out, None, None, None, 0, 0, 0))
|
||||
fake_torch_npu = _fake_torch_npu(monkeypatch, npu_fusion_attention=fusion_attention)
|
||||
impl = FlashAttentionImpl(num_heads=2, head_size=4, softmax_scale=0.5, causal=True)
|
||||
query = torch.randn(1, 4, 2, 4)
|
||||
key_keep_mask = torch.tensor([[True, False, True, True]])
|
||||
|
||||
out = impl.forward_fa_npu(
|
||||
query,
|
||||
query,
|
||||
query,
|
||||
AttentionMetadata(attn_mask=key_keep_mask),
|
||||
)
|
||||
|
||||
fake_torch_npu.npu_fusion_attention.assert_called_once()
|
||||
kwargs = fake_torch_npu.npu_fusion_attention.call_args.kwargs
|
||||
assert kwargs["sparse_mode"] == 1
|
||||
assert kwargs["inner_precise"] == 2
|
||||
expected_block_mask = torch.tensor(
|
||||
[
|
||||
[
|
||||
[
|
||||
[False, True, True, True],
|
||||
[False, True, True, True],
|
||||
[False, True, False, True],
|
||||
[False, True, False, False],
|
||||
]
|
||||
]
|
||||
]
|
||||
)
|
||||
assert torch.equal(kwargs["atten_mask"], expected_block_mask)
|
||||
assert kwargs["atten_mask"].is_contiguous()
|
||||
assert out is expected_out
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_npu_noncausal_without_explicit_mask_stays_unmasked(monkeypatch):
|
||||
captured: dict = {}
|
||||
|
||||
def fake_attention_forward(query, key, value, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return torch.zeros_like(query)
|
||||
|
||||
_fake_mindiesd(monkeypatch, attention_forward=fake_attention_forward)
|
||||
query = torch.randn(1, 4, 2, 4)
|
||||
|
||||
_npu_impl(causal=False).forward_fa_npu(query, query, query)
|
||||
|
||||
assert captured["attn_mask"] is None
|
||||
|
||||
|
||||
# --- Test group D: laser input pre-scaling in _forward_prefix_kv_slice_npu --
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_prefix_kv_slice_applies_laser_input_scaling(monkeypatch):
|
||||
monkeypatch.setenv("MINDIE_SD_FA_TYPE", "ascend_laser_attention")
|
||||
captured: dict = {}
|
||||
@@ -726,6 +860,7 @@ def test_prefix_kv_slice_applies_laser_input_scaling(monkeypatch):
|
||||
torch.testing.assert_close(out, torch.ones(1, 8, 2, 4) * 256.0)
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_prefix_kv_slice_no_scaling_without_factor(monkeypatch):
|
||||
monkeypatch.setenv("MINDIE_SD_FA_TYPE", "ascend_laser_attention")
|
||||
captured: dict = {}
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionImpl
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.npu]
|
||||
|
||||
|
||||
def _bottom_right_causal_reference(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
scale: float,
|
||||
key_keep_mask: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
scores = torch.einsum("bqhd,bkhd->bhqk", query.float(), key.float()) * scale
|
||||
query_positions = torch.arange(query.shape[1]).unsqueeze(1)
|
||||
key_positions = torch.arange(key.shape[1]).unsqueeze(0)
|
||||
keep_mask = key_positions <= query_positions + (key.shape[1] - query.shape[1])
|
||||
if key_keep_mask is not None:
|
||||
assert query.shape[0] == 1
|
||||
keep_mask = keep_mask & key_keep_mask[0].bool().unsqueeze(0)
|
||||
valid_rows = keep_mask.any(dim=-1)
|
||||
|
||||
output = torch.zeros(
|
||||
query.shape[0],
|
||||
query.shape[1],
|
||||
query.shape[2],
|
||||
value.shape[-1],
|
||||
dtype=torch.float32,
|
||||
)
|
||||
valid_scores = scores[:, :, valid_rows].masked_fill(~keep_mask[valid_rows], float("-inf"))
|
||||
probabilities = torch.softmax(valid_scores, dim=-1)
|
||||
output[:, valid_rows] = torch.einsum("bhqk,bkhd->bqhd", probabilities, value.float())
|
||||
return output
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_omni_platform.is_npu(), reason="Native causal attention requires Ascend NPU")
|
||||
def test_npu_bottom_right_causal_fully_masked_rows_are_zero():
|
||||
torch.manual_seed(0)
|
||||
batch_size, query_length, key_length = 1, 4, 2
|
||||
num_heads, head_size = 2, 64
|
||||
scale = head_size**-0.5
|
||||
|
||||
query_cpu = torch.randn(batch_size, query_length, num_heads, head_size).to(torch.bfloat16)
|
||||
key_cpu = torch.randn(batch_size, key_length, num_heads, head_size).to(torch.bfloat16)
|
||||
value_cpu = torch.randn(batch_size, key_length, num_heads, head_size).to(torch.bfloat16)
|
||||
expected = _bottom_right_causal_reference(query_cpu, key_cpu, value_cpu, scale)
|
||||
|
||||
device = torch.device(current_omni_platform.device_type)
|
||||
query = query_cpu.to(device)
|
||||
key = key_cpu.to(device)
|
||||
value = value_cpu.to(device)
|
||||
impl = FlashAttentionImpl(
|
||||
num_heads=num_heads,
|
||||
head_size=head_size,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
)
|
||||
|
||||
output = impl.forward_fa_npu(query, key, value)
|
||||
output_cpu = output.float().cpu()
|
||||
|
||||
assert torch.isfinite(output_cpu).all()
|
||||
assert torch.equal(output_cpu[:, : query_length - key_length], torch.zeros_like(output_cpu[:, :2]))
|
||||
torch.testing.assert_close(
|
||||
output_cpu[:, query_length - key_length :],
|
||||
expected[:, query_length - key_length :],
|
||||
rtol=2e-2,
|
||||
atol=2e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_omni_platform.is_npu(), reason="Native causal attention requires Ascend NPU")
|
||||
def test_npu_bottom_right_causal_composes_explicit_keep_mask():
|
||||
torch.manual_seed(1)
|
||||
batch_size, sequence_length = 1, 4
|
||||
num_heads, head_size = 2, 64
|
||||
scale = head_size**-0.5
|
||||
key_keep_mask_cpu = torch.tensor([[True, False, True, True]])
|
||||
|
||||
query_cpu = torch.randn(batch_size, sequence_length, num_heads, head_size).to(torch.bfloat16)
|
||||
key_cpu = torch.randn(batch_size, sequence_length, num_heads, head_size).to(torch.bfloat16)
|
||||
value_cpu = torch.randn(batch_size, sequence_length, num_heads, head_size).to(torch.bfloat16)
|
||||
expected = _bottom_right_causal_reference(
|
||||
query_cpu,
|
||||
key_cpu,
|
||||
value_cpu,
|
||||
scale,
|
||||
key_keep_mask_cpu,
|
||||
)
|
||||
|
||||
device = torch.device(current_omni_platform.device_type)
|
||||
query = query_cpu.to(device)
|
||||
key = key_cpu.to(device)
|
||||
value = value_cpu.to(device)
|
||||
impl = FlashAttentionImpl(
|
||||
num_heads=num_heads,
|
||||
head_size=head_size,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
)
|
||||
|
||||
output = impl.forward_fa_npu(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
AttentionMetadata(attn_mask=key_keep_mask_cpu.to(device)),
|
||||
)
|
||||
output_cpu = output.float().cpu()
|
||||
|
||||
assert torch.isfinite(output_cpu).all()
|
||||
torch.testing.assert_close(output_cpu, expected, rtol=2e-2, atol=2e-2)
|
||||
@@ -0,0 +1,136 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.diffusion.attention import layer
|
||||
from vllm_omni.diffusion.attention.backends.fastvideo_vsa import FastVideoVSABackend, FastVideoVSAImpl
|
||||
from vllm_omni.diffusion.attention.backends.sdpa import SDPABackend
|
||||
from vllm_omni.diffusion.attention.parallel.base import NoParallelAttention
|
||||
from vllm_omni.diffusion.data import AttentionSpec
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _SpecializedImpl(FastVideoVSAImpl):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_attention(monkeypatch):
|
||||
monkeypatch.setattr(layer, "get_current_diffusion_config_or_none", lambda: None)
|
||||
monkeypatch.setattr(layer, "build_parallel_attention_strategy", lambda **kwargs: NoParallelAttention())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("explicit", [False, True])
|
||||
@pytest.mark.parametrize("override", [False, True])
|
||||
def test_specialization_preserves_selected_backend_and_options(mocker, local_attention, explicit, override):
|
||||
spec = AttentionSpec(backend="FASTVIDEO_VSA", fastvideo_vsa_topk=7) if explicit else None
|
||||
select = mocker.patch.object(layer, "get_attn_backend_for_role", return_value=(FastVideoVSABackend, spec))
|
||||
attention = layer.Attention(
|
||||
num_heads=2,
|
||||
head_size=8,
|
||||
softmax_scale=8**-0.5,
|
||||
causal=False,
|
||||
role="video.self",
|
||||
role_category="self",
|
||||
qkv_layout="BSND",
|
||||
impl_overrides={"FASTVIDEO_VSA": _SpecializedImpl} if override else None,
|
||||
)
|
||||
|
||||
assert attention.attn_backend is FastVideoVSABackend
|
||||
assert type(attention.attention) is (_SpecializedImpl if override else FastVideoVSAImpl)
|
||||
assert attention.attention.topk == (7 if explicit else 64)
|
||||
assert attention.attention.qkv_layout == "BSND"
|
||||
assert attention.backend_pref == "FASTVIDEO_VSA"
|
||||
assert attention.backend_explicit is explicit
|
||||
assert attention.attn_spec is spec
|
||||
assert isinstance(attention.parallel_strategy, NoParallelAttention)
|
||||
select.assert_called_once_with(
|
||||
role="video.self", head_size=8, attention_config=None, role_category="self", allow_trtllm_default=False
|
||||
)
|
||||
|
||||
|
||||
def test_unselected_specialization_does_not_replace_dense_backend(mocker, local_attention):
|
||||
mocker.patch.object(layer, "get_attn_backend_for_role", return_value=(SDPABackend, None))
|
||||
attention = layer.Attention(
|
||||
num_heads=2,
|
||||
head_size=8,
|
||||
softmax_scale=8**-0.5,
|
||||
causal=False,
|
||||
impl_overrides={"FASTVIDEO_VSA": _SpecializedImpl},
|
||||
)
|
||||
assert attention.attn_backend is SDPABackend
|
||||
assert type(attention.attention) is SDPABackend.get_impl_cls()
|
||||
|
||||
|
||||
def test_specialization_does_not_bypass_selection_errors(mocker, local_attention):
|
||||
mocker.patch.object(layer, "get_attn_backend_for_role", side_effect=ImportError("optional kernel unavailable"))
|
||||
with pytest.raises(ImportError, match="optional kernel unavailable"):
|
||||
layer.Attention(
|
||||
num_heads=2,
|
||||
head_size=8,
|
||||
softmax_scale=8**-0.5,
|
||||
causal=False,
|
||||
impl_overrides={"FASTVIDEO_VSA": _SpecializedImpl},
|
||||
)
|
||||
|
||||
|
||||
def test_specialization_must_extend_selected_implementation(mocker, local_attention):
|
||||
mocker.patch.object(layer, "get_attn_backend_for_role", return_value=(FastVideoVSABackend, None))
|
||||
with pytest.raises(TypeError, match="must subclass the selected implementation"):
|
||||
layer.Attention(
|
||||
num_heads=2,
|
||||
head_size=8,
|
||||
softmax_scale=8**-0.5,
|
||||
causal=False,
|
||||
impl_overrides={"FASTVIDEO_VSA": SDPABackend.get_impl_cls()},
|
||||
)
|
||||
|
||||
|
||||
def test_platform_backend_capabilities_survive_model_specialization(mocker, local_attention):
|
||||
from vllm_omni.diffusion.models.minimax_h3.attention.fastvideo_h3 import MiniMaxH3VSAImpl
|
||||
|
||||
class PlatformBackend(FastVideoVSABackend):
|
||||
@classmethod
|
||||
def supports_packed_mask_free(cls):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_supported_head_sizes():
|
||||
return [128]
|
||||
|
||||
# A registry/platform may refine capabilities without changing the
|
||||
# implementation. The model must not replace those capability decisions.
|
||||
mocker.patch.object(layer, "get_attn_backend_for_role", return_value=(PlatformBackend, None))
|
||||
attention = layer.Attention(
|
||||
num_heads=2,
|
||||
head_size=128,
|
||||
softmax_scale=128**-0.5,
|
||||
causal=False,
|
||||
impl_overrides={"FASTVIDEO_VSA": MiniMaxH3VSAImpl},
|
||||
)
|
||||
assert attention.attn_backend is PlatformBackend
|
||||
assert not attention.attn_backend.supports_packed_mask_free()
|
||||
assert attention.attn_backend.get_supported_head_sizes() == [128]
|
||||
assert type(attention.attention) is MiniMaxH3VSAImpl
|
||||
|
||||
|
||||
def test_incompatible_platform_implementation_is_not_discarded(mocker, local_attention):
|
||||
class PlatformImpl(FastVideoVSAImpl):
|
||||
pass
|
||||
|
||||
class PlatformBackend(FastVideoVSABackend):
|
||||
@staticmethod
|
||||
def get_impl_cls():
|
||||
return PlatformImpl
|
||||
|
||||
mocker.patch.object(layer, "get_attn_backend_for_role", return_value=(PlatformBackend, None))
|
||||
with pytest.raises(TypeError, match="selected implementation .*PlatformImpl"):
|
||||
layer.Attention(
|
||||
num_heads=2,
|
||||
head_size=8,
|
||||
softmax_scale=8**-0.5,
|
||||
causal=False,
|
||||
impl_overrides={"FASTVIDEO_VSA": _SpecializedImpl},
|
||||
)
|
||||
@@ -0,0 +1,160 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Cross-repository CPU dispatch contracts, requiring the matching MindIE-SD source.
|
||||
|
||||
Only native operators are replaced. Public adapters, rotations, layouts, padding,
|
||||
scale preparation, sequence metadata and output cropping execute their real code.
|
||||
Operator substitutes carry floating tensors; these are not numerical quantization tests.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.attention.backends import flash_attn, rainfusion_attn
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.platforms.npu.quant.kv_quant_npu import get_quant_attention_rotation
|
||||
|
||||
mindiesd = pytest.importorskip("mindiesd")
|
||||
torch_npu = importlib.import_module("torch_npu")
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"precision,batch,q_len,kv_len,q_heads",
|
||||
[
|
||||
("fp8", 1, 130, 130, 2),
|
||||
("mxfp8", 1, 130, 130, 2),
|
||||
("mxfp4", 1, 130, 130, 2),
|
||||
("mxfp8", 2, 17, 8, 4),
|
||||
("mxfp8", 2, 8, 17, 4),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("layout", ["BSND", "BNSD"])
|
||||
def test_omni_calls_real_dense_public_api(monkeypatch, precision, layout, batch, q_len, kv_len, q_heads):
|
||||
assert callable(getattr(mindiesd, "quant_attention", None)), "MindIE-SD must export quant_attention"
|
||||
monkeypatch.setattr(flash_attn, "current_omni_platform", SimpleNamespace(is_npu=lambda: True, device_name="npu"))
|
||||
monkeypatch.setattr(flash_attn, "get_current_diffusion_config_or_none", lambda: None)
|
||||
quantized_inputs = []
|
||||
|
||||
def dynamic_quant(tensor, **kwargs):
|
||||
quantized_inputs.append((tensor.clone(), kwargs))
|
||||
shape = list(tensor.shape)
|
||||
axis = kwargs.get("axis", -1)
|
||||
shape[axis] = max(1, shape[axis] // 32)
|
||||
return tensor, torch.ones(shape, dtype=torch.uint8)
|
||||
|
||||
monkeypatch.setattr(torch_npu, "npu_dynamic_block_quant", dynamic_quant, raising=False)
|
||||
monkeypatch.setattr(torch_npu, "npu_dynamic_mx_quant", dynamic_quant, raising=False)
|
||||
execute = Mock(side_effect=lambda query, *args, **kwargs: (query, None))
|
||||
metadata = Mock(return_value=torch.zeros(1, dtype=torch.int32))
|
||||
if precision == "fp8":
|
||||
monkeypatch.setattr(torch.ops.mindiesd, "fused_infer_attention_score_v2", execute, raising=False)
|
||||
elif precision == "mxfp8":
|
||||
monkeypatch.setattr(torch_npu, "npu_fused_infer_attention_score_v2", execute, raising=False)
|
||||
else:
|
||||
monkeypatch.setattr(torch.ops.mindiesd, "quant_flash_attn", execute, raising=False)
|
||||
monkeypatch.setattr(torch.ops.mindiesd, "quant_flash_attn_metadata", metadata, raising=False)
|
||||
|
||||
query = torch.randn(batch, q_len, q_heads, 64, dtype=torch.bfloat16)
|
||||
key = torch.randn(batch, kv_len, 2, 64, dtype=torch.bfloat16)
|
||||
value = torch.randn_like(key)
|
||||
if layout == "BNSD":
|
||||
query, key, value = (tensor.transpose(1, 2) for tensor in (query, key, value))
|
||||
impl = flash_attn.FlashAttentionImpl(
|
||||
num_heads=q_heads,
|
||||
head_size=64,
|
||||
softmax_scale=0.37,
|
||||
qkv_layout=layout,
|
||||
)
|
||||
output = impl.forward_npu(query, key, value, AttentionMetadata(extra={"kv_cache_dtype": precision}))
|
||||
rotation = get_quant_attention_rotation(query.device, query.dtype, 64)
|
||||
expected = query @ rotation
|
||||
torch.testing.assert_close(output, expected)
|
||||
assert output.shape == query.shape and len(quantized_inputs) == 3
|
||||
execute.assert_called_once()
|
||||
kwargs = execute.call_args.kwargs
|
||||
assert kwargs["softmax_scale"] == 0.37
|
||||
if precision == "fp8":
|
||||
assert kwargs["input_layout"] == "BNSD"
|
||||
assert quantized_inputs[0][0].shape == (2, 130, 64)
|
||||
assert quantized_inputs[0][1]["row_block_size"] == 128
|
||||
elif precision == "mxfp8":
|
||||
args = execute.call_args.args
|
||||
assert args[0].shape == (batch * q_len, q_heads, 64)
|
||||
assert args[1].shape == args[2].shape == (batch * kv_len, 2, 64)
|
||||
assert kwargs["input_layout"] == "TND"
|
||||
assert kwargs["num_query_heads"] == q_heads and kwargs["num_key_value_heads"] == 2
|
||||
assert kwargs["actual_seq_qlen"] == [q_len * (i + 1) for i in range(batch)]
|
||||
assert kwargs["actual_seq_kvlen"] == [kv_len * (i + 1) for i in range(batch)]
|
||||
assert [entry[1]["axis"] for entry in quantized_inputs] == [-1, -1, 0]
|
||||
else:
|
||||
seq_axis = 1 if layout == "BSND" else 2
|
||||
assert quantized_inputs[0][0].shape[seq_axis] == 512
|
||||
assert kwargs["seqused_q"].tolist() == kwargs["seqused_kv"].tolist() == [512]
|
||||
assert kwargs["layout_q"] == kwargs["layout_out"] == layout
|
||||
assert kwargs["q_dtype"] == torch_npu.float4_e2m1fn_x2
|
||||
assert kwargs["q_descale_dtype"] == torch_npu.float8_e8m0fnu
|
||||
assert execute.call_args.args[3].shape[:2] == (1, 2)
|
||||
assert [entry[1]["axis"] for entry in quantized_inputs] == [-1, -1, seq_axis]
|
||||
assert kwargs["metadata"] is metadata.return_value
|
||||
metadata.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("precision", ["fp8", "mxfp4"])
|
||||
@pytest.mark.parametrize("length", [65, 600])
|
||||
def test_omni_calls_real_sparse_public_api(monkeypatch, precision, length):
|
||||
rf = importlib.import_module("mindiesd.layers.flash_attn.sparse_flash_attn_rf_v3")
|
||||
monkeypatch.setattr(flash_attn, "current_omni_platform", SimpleNamespace(is_npu=lambda: True, device_name="npu"))
|
||||
for module in (flash_attn, rainfusion_attn):
|
||||
monkeypatch.setattr(module, "get_current_diffusion_config_or_none", lambda: None)
|
||||
rainfusion_attn._mindiesd_supports_precision.cache_clear()
|
||||
monkeypatch.setattr(rf, "do_tensor_rearrange_only", lambda q, k, v, *a, **kw: (q, k, v))
|
||||
monkeypatch.setattr(rf, "avgpool", lambda *a, **kw: torch.ones(1))
|
||||
monkeypatch.setattr(rf, "_generate_mask_direct", lambda *a, **kw: torch.ones(1, 2, 8, 8))
|
||||
monkeypatch.setattr(rf, "_bsa_inv_rearrange", lambda out, *a: out)
|
||||
|
||||
def quantize(tensor, **kwargs):
|
||||
shape = list(tensor.shape)
|
||||
axis = kwargs.get("axis", -1)
|
||||
shape[axis] = max(1, shape[axis] // 32)
|
||||
return tensor, torch.ones(shape, dtype=torch.uint8)
|
||||
|
||||
monkeypatch.setattr(torch_npu, "npu_dynamic_mx_quant", quantize, raising=False)
|
||||
monkeypatch.setattr(torch_npu, "npu_dynamic_block_quant", quantize, raising=False)
|
||||
execute = Mock(side_effect=lambda **kw: (torch.ones_like(kw["query"]), None))
|
||||
monkeypatch.setattr(torch.ops.mindiesd, "block_sparse_attention", execute, raising=False)
|
||||
query = torch.randn(1, length, 2, 64, dtype=torch.bfloat16)
|
||||
impl = rainfusion_attn.RainFusionAttentionImpl(
|
||||
num_heads=2,
|
||||
head_size=64,
|
||||
softmax_scale=0.37,
|
||||
qkv_layout="BSND",
|
||||
backend_kwargs={"sparsity": 0.8},
|
||||
)
|
||||
plan = rainfusion_attn.RainFusionPlan(used_len=length, prefix_len=0, latent_shape=(1, 1, length))
|
||||
try:
|
||||
output = impl._forward_sparse_npu(
|
||||
query,
|
||||
query,
|
||||
query,
|
||||
plan,
|
||||
AttentionMetadata(extra={"kv_cache_dtype": precision}),
|
||||
)
|
||||
finally:
|
||||
rainfusion_attn._mindiesd_supports_precision.cache_clear()
|
||||
torch.testing.assert_close(output, torch.ones_like(query))
|
||||
execute.assert_called_once()
|
||||
kwargs = execute.call_args.kwargs
|
||||
assert kwargs["actual_seq_lengths"] == kwargs["actual_seq_lengths_kv"] == [length]
|
||||
assert kwargs["q_input_layout"] == "BNSD"
|
||||
assert kwargs["block_sparse_mask"] is not None
|
||||
if precision == "mxfp4":
|
||||
assert kwargs["query"].shape[2] == (length + 63) // 64 * 64
|
||||
assert kwargs["quant_mode"] == 2
|
||||
assert kwargs["q_dtype"] == torch_npu.float4_e2m1fn_x2
|
||||
assert kwargs["v_scale_dtype"] == torch_npu.float8_e8m0fnu
|
||||
@@ -0,0 +1,323 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""CPU contract tests for MindIE adapters; no NPU kernels are mocked in production."""
|
||||
|
||||
import sys
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.attention import layer as layer_mod
|
||||
from vllm_omni.diffusion.attention.backends import flash_attn, rainfusion_attn
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata, VideoTokenLayout
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
_DEFAULT_SKIP_STEPS = frozenset({0, 1, 38, 39})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def runtime(monkeypatch):
|
||||
mod = ModuleType("mindiesd")
|
||||
|
||||
def execute(q, k, v, *, precision, layout, scale, q_rot=None, k_rot=None):
|
||||
return q
|
||||
|
||||
setattr(mod, "quant_attention", Mock(side_effect=execute))
|
||||
monkeypatch.setitem(sys.modules, mod.__name__, mod)
|
||||
for module in (flash_attn, layer_mod):
|
||||
monkeypatch.setattr(module, "current_omni_platform", SimpleNamespace(is_npu=lambda: True, device_name="npu"))
|
||||
monkeypatch.setattr(flash_attn, "get_current_diffusion_config_or_none", lambda: None)
|
||||
monkeypatch.setattr(rainfusion_attn, "get_current_diffusion_config_or_none", lambda: None)
|
||||
monkeypatch.setattr(rainfusion_attn, "is_forward_context_available", lambda: False)
|
||||
supports_precision = rainfusion_attn._mindiesd_supports_precision
|
||||
supports_precision.cache_clear()
|
||||
yield mod
|
||||
supports_precision.cache_clear()
|
||||
|
||||
|
||||
def flash(layout="BSND"):
|
||||
return flash_attn.FlashAttentionImpl(
|
||||
num_heads=2,
|
||||
head_size=64,
|
||||
softmax_scale=0.37,
|
||||
qkv_layout=layout,
|
||||
)
|
||||
|
||||
|
||||
def sparse(**kwargs):
|
||||
return rainfusion_attn.RainFusionAttentionImpl(
|
||||
num_heads=2,
|
||||
head_size=64,
|
||||
softmax_scale=0.37,
|
||||
qkv_layout="BSND",
|
||||
backend_kwargs={"sparsity": 0.8, **kwargs},
|
||||
)
|
||||
|
||||
|
||||
def video_metadata(**extra):
|
||||
return AttentionMetadata(
|
||||
video_layout=VideoTokenLayout(prefix_len=0, latent_grid=(4, 32, 32), used_len=4096),
|
||||
extra={"max_seqlen_q": 4096, **extra},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["fp8", "mxfp8", "mxfp4"])
|
||||
@pytest.mark.parametrize("layout", [None, "BSND", "BNSD"])
|
||||
def test_exact_runtime_layout_scale_and_rotation(runtime, method, layout):
|
||||
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
|
||||
if layout == "BNSD":
|
||||
q = q.transpose(1, 2)
|
||||
impl = flash(layout=layout)
|
||||
metadata = AttentionMetadata(extra={"kv_cache_dtype": method})
|
||||
assert impl.forward_npu(q, q, q, metadata) is q
|
||||
args, kwargs = runtime.quant_attention.call_args
|
||||
assert args[0] is q # no unconditional transpose or extra quantization
|
||||
assert kwargs["layout"] == (layout or "BSND") and kwargs["scale"] == 0.37 and kwargs["precision"] == method
|
||||
assert "attn_mask" not in kwargs
|
||||
from vllm_omni.platforms.npu.quant.kv_quant_npu import get_quant_attention_rotation
|
||||
|
||||
assert kwargs["q_rot"] is kwargs["k_rot"]
|
||||
torch.testing.assert_close(kwargs["q_rot"], get_quant_attention_rotation(q.device, q.dtype, 64))
|
||||
|
||||
|
||||
def test_missing_runtime_requires_config_change(runtime):
|
||||
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
|
||||
del runtime.quant_attention
|
||||
with pytest.raises(ImportError, match="diffusion_kv_cache_dtype"):
|
||||
flash().forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": "mxfp4"}))
|
||||
|
||||
|
||||
def test_operator_failure_never_falls_back(runtime):
|
||||
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
|
||||
runtime.quant_attention.side_effect = RuntimeError("operator failure")
|
||||
with pytest.raises(RuntimeError, match="operator failure"):
|
||||
flash().forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": "mxfp4"}))
|
||||
runtime.quant_attention.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["fp8", "mxfp8", "mxfp4"])
|
||||
def test_unsupported_shape_requires_config_change(runtime, method):
|
||||
q = torch.randn(1, 128, 2, 96, dtype=torch.bfloat16)
|
||||
with pytest.raises(ValueError, match="diffusion_kv_cache_dtype"):
|
||||
flash().forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": method}))
|
||||
runtime.quant_attention.assert_not_called()
|
||||
|
||||
|
||||
def test_packed_metadata_never_reaches_quant_runtime(runtime):
|
||||
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
|
||||
metadata = AttentionMetadata(extra={"kv_cache_dtype": "mxfp8", "cu_seqlens_q": torch.tensor([0, 64, 128])})
|
||||
with pytest.raises(ValueError, match="packed/varlen"):
|
||||
flash().forward_npu(q, q, q, metadata)
|
||||
runtime.quant_attention.assert_not_called()
|
||||
|
||||
|
||||
def make_layer(runtime, monkeypatch, method="mxfp8", skip_steps=_DEFAULT_SKIP_STEPS):
|
||||
layer = object.__new__(layer_mod.Attention)
|
||||
torch.nn.Module.__init__(layer)
|
||||
layer.attention = flash()
|
||||
layer.attn_backend = flash_attn.FlashAttentionBackend
|
||||
layer._disable_kv_quant = False
|
||||
layer.layer_idx = 3
|
||||
cfg = SimpleNamespace(
|
||||
diffusion_kv_cache_dtype=method,
|
||||
diffusion_kv_cache_skip_step_indices=skip_steps,
|
||||
diffusion_kv_cache_skip_layer_indices=None,
|
||||
parallel_config=SimpleNamespace(ring_degree=1),
|
||||
)
|
||||
layer._init_kv_cache_quantization(cfg)
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["fp8", "mxfp8", "mxfp4"])
|
||||
def test_global_quant_method_is_published_per_forward(runtime, monkeypatch, method):
|
||||
layer = make_layer(runtime, monkeypatch, method=method, skip_steps=None)
|
||||
assert layer._with_kv_cache_dtype(None).extra["kv_cache_dtype"] == method
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", [None, "auto", "float"])
|
||||
def test_optout_without_quant_policy_keeps_metadata_unchanged(runtime, monkeypatch, method):
|
||||
layer = make_layer(runtime, monkeypatch)
|
||||
layer._disable_kv_quant = True
|
||||
layer._init_kv_cache_quantization(
|
||||
SimpleNamespace(
|
||||
diffusion_kv_cache_dtype=method,
|
||||
diffusion_kv_cache_skip_step_indices=None,
|
||||
diffusion_kv_cache_skip_layer_indices=None,
|
||||
parallel_config=SimpleNamespace(ring_degree=1),
|
||||
)
|
||||
)
|
||||
source = video_metadata()
|
||||
assert layer._with_kv_cache_dtype(source) is source
|
||||
|
||||
|
||||
def test_step_skip_is_request_local_and_cross_optout(runtime, monkeypatch):
|
||||
layer = make_layer(runtime, monkeypatch)
|
||||
ctx = SimpleNamespace(denoise_step_idx=0)
|
||||
monkeypatch.setattr(layer_mod, "is_forward_context_available", lambda: True)
|
||||
monkeypatch.setattr(layer_mod, "get_forward_context", lambda: ctx)
|
||||
source = video_metadata()
|
||||
for step in list(range(40)) * 2:
|
||||
ctx.denoise_step_idx = step
|
||||
resolved = layer._with_kv_cache_dtype(source)
|
||||
assert resolved.extra["kv_cache_dtype"] == ("float" if step in (0, 1, 38, 39) else "mxfp8")
|
||||
assert resolved.video_layout is source.video_layout
|
||||
assert source.extra == {"max_seqlen_q": 4096}
|
||||
layer._disable_kv_quant = True
|
||||
assert layer._with_kv_cache_dtype(source).extra["kv_cache_dtype"] == "float"
|
||||
|
||||
|
||||
def test_ring_attention_is_rejected(runtime, monkeypatch):
|
||||
layer = make_layer(runtime, monkeypatch)
|
||||
cfg = SimpleNamespace(diffusion_kv_cache_dtype="mxfp8", parallel_config=SimpleNamespace(ring_degree=2))
|
||||
with pytest.raises(ValueError, match="ring"):
|
||||
layer._init_kv_cache_quantization(cfg)
|
||||
|
||||
|
||||
def test_sparse_tail_steps_and_dense_quant_dispatch(runtime, monkeypatch):
|
||||
ctx = SimpleNamespace(denoise_step_idx=49, total_denoise_steps=50)
|
||||
monkeypatch.setattr(rainfusion_attn, "is_forward_context_available", lambda: True)
|
||||
monkeypatch.setattr(rainfusion_attn, "get_forward_context", lambda: ctx)
|
||||
impl = sparse(start_step=2, end_step=2)
|
||||
metadata = video_metadata(kv_cache_dtype="fp8")
|
||||
assert impl._resolve_plan(metadata) is None
|
||||
q = torch.randn(1, 4096, 2, 64, dtype=torch.bfloat16)
|
||||
impl.forward_npu(q, q, q, metadata)
|
||||
assert runtime.quant_attention.call_args.kwargs["precision"] == "fp8"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["fp8", "mxfp4"])
|
||||
def test_sparse_quant_uses_public_runtime(runtime, monkeypatch, method):
|
||||
runtime.sparse_attention = Mock(side_effect=lambda q, k, v, **kwargs: q)
|
||||
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: True)
|
||||
q = torch.randn(1, 4224, 2, 64, dtype=torch.bfloat16)
|
||||
out = sparse().forward_npu(q, q, q, video_metadata(kv_cache_dtype=method))
|
||||
# The sparse path reuses the query tail as the output padding instead of
|
||||
# zero-filling a fresh buffer; rows past the 4096-row valid prefix are
|
||||
# never read back downstream (the model unpads by cu_seqlens), so pin the
|
||||
# passthrough rather than requiring zeros.
|
||||
assert out.shape == q.shape and torch.equal(out[:, 4096:], q[:, 4096:])
|
||||
kwargs = runtime.sparse_attention.call_args.kwargs
|
||||
assert (kwargs["precision"], kwargs["sparse_type"], kwargs["inner_precise"]) == (method, "rf_v3", 4)
|
||||
runtime.quant_attention.assert_not_called()
|
||||
|
||||
|
||||
def test_sparse_custom_mask_stays_dense(runtime):
|
||||
metadata = AttentionMetadata(
|
||||
attn_mask=torch.ones(1, 4096, dtype=torch.bool),
|
||||
video_layout=VideoTokenLayout(prefix_len=0, latent_grid=(4, 32, 32)),
|
||||
extra={"max_seqlen_q": 4096},
|
||||
)
|
||||
assert sparse()._resolve_plan(metadata) is None
|
||||
|
||||
|
||||
def test_sparse_model_padding_mask_is_equivalent_to_trimming(runtime):
|
||||
metadata = video_metadata()
|
||||
metadata.attn_mask = torch.arange(4224)[None] < 4096
|
||||
assert sparse()._resolve_plan(metadata).used_len == 4096
|
||||
|
||||
|
||||
def test_bsa_operator_failure_never_retries(runtime, monkeypatch):
|
||||
runtime.sparse_attention = Mock(side_effect=RuntimeError("BSA operator failure"))
|
||||
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: True)
|
||||
q = torch.randn(1, 4096, 2, 64, dtype=torch.bfloat16)
|
||||
with pytest.raises(RuntimeError, match="BSA operator failure"):
|
||||
sparse().forward_npu(q, q, q, video_metadata(kv_cache_dtype="mxfp4"))
|
||||
runtime.sparse_attention.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"method,shape,supports_precision,error",
|
||||
[
|
||||
("mxfp8", (1, 4096, 2, 64), True, "sparse mxfp8 is not supported"),
|
||||
("mxfp4", (1, 4096, 2, 64), False, "explicitly support precision"),
|
||||
("fp8", (2, 4096, 2, 64), True, "batch size 1"),
|
||||
("fp8", (1, 4096, 2, 32), True, "power-of-two head dimension"),
|
||||
("mxfp4", (1, 4096, 2, 96), True, "power-of-two head dimension"),
|
||||
],
|
||||
)
|
||||
def test_invalid_sparse_quant_requires_config_change(runtime, monkeypatch, method, shape, supports_precision, error):
|
||||
runtime.sparse_attention = Mock(side_effect=lambda q, k, v, **kwargs: q)
|
||||
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: supports_precision)
|
||||
q = torch.randn(shape, dtype=torch.bfloat16)
|
||||
with pytest.raises(ValueError, match=error):
|
||||
sparse().forward_npu(q, q, q, video_metadata(kv_cache_dtype=method))
|
||||
runtime.sparse_attention.assert_not_called()
|
||||
|
||||
|
||||
def test_layer_selector_requires_index_except_cross_optout(runtime, monkeypatch):
|
||||
layer = make_layer(runtime, monkeypatch)
|
||||
layer.layer_idx = None
|
||||
config = SimpleNamespace(
|
||||
diffusion_kv_cache_dtype="mxfp8",
|
||||
diffusion_kv_cache_skip_step_indices=None,
|
||||
diffusion_kv_cache_skip_layer_indices={0, 3},
|
||||
parallel_config=SimpleNamespace(ring_degree=1),
|
||||
)
|
||||
with pytest.raises(ValueError, match="parseable transformer block index"):
|
||||
layer._init_kv_cache_quantization(config)
|
||||
layer._disable_kv_quant = True
|
||||
layer._init_kv_cache_quantization(config)
|
||||
assert layer._with_kv_cache_dtype(None).extra["kv_cache_dtype"] == "float"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prefix", ["transformer.blocks.3.attn1", "transformer_2.blocks.3.attn1"])
|
||||
@pytest.mark.parametrize("method", ["fp8", "mxfp4"])
|
||||
def test_expert_layer_and_step_skip_remain_sparse(runtime, monkeypatch, prefix, method):
|
||||
def execute(q, k, v, *, precision="bf16", **kwargs):
|
||||
return q
|
||||
|
||||
runtime.sparse_attention = Mock(wraps=execute)
|
||||
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: True)
|
||||
layer = make_layer(runtime, monkeypatch)
|
||||
layer.layer_idx = rainfusion_attn._try_extract_layer_index(prefix)
|
||||
assert layer.layer_idx == 3
|
||||
config = SimpleNamespace(
|
||||
diffusion_kv_cache_dtype=method,
|
||||
diffusion_kv_cache_skip_step_indices={0, 1, 38, 39},
|
||||
diffusion_kv_cache_skip_layer_indices={3},
|
||||
parallel_config=SimpleNamespace(ring_degree=1),
|
||||
)
|
||||
impl = sparse()
|
||||
layer.attention = impl
|
||||
layer.attn_backend = rainfusion_attn.RainFusionAttentionBackend
|
||||
layer._init_kv_cache_quantization(config)
|
||||
ctx = SimpleNamespace(denoise_step_idx=0)
|
||||
monkeypatch.setattr(layer_mod, "is_forward_context_available", lambda: True)
|
||||
monkeypatch.setattr(layer_mod, "get_forward_context", lambda: ctx)
|
||||
source = video_metadata(kv_cache_dtype="mxfp8")
|
||||
q = torch.randn(1, 4096, 2, 64, dtype=torch.bfloat16)
|
||||
for layer_idx, step in ((3, 0), (3, 2), (4, 0), (4, 1), (4, 2), (4, 37), (4, 38), (4, 39), (4, 0)):
|
||||
expected = "bf16" if layer_idx == 3 or step in (0, 1, 38, 39) else method
|
||||
layer.layer_idx = layer_idx
|
||||
ctx.denoise_step_idx = step
|
||||
metadata = layer._with_kv_cache_dtype(source)
|
||||
impl.forward_npu(q, q, q, metadata)
|
||||
assert runtime.sparse_attention.call_args.kwargs["precision"] == expected
|
||||
runtime.quant_attention.assert_not_called()
|
||||
assert source.extra["kv_cache_dtype"] == "mxfp8"
|
||||
|
||||
|
||||
def test_custom_attention_initialization_keeps_upstream_optout(monkeypatch):
|
||||
monkeypatch.setattr(layer_mod, "get_current_diffusion_config_or_none", lambda: None)
|
||||
monkeypatch.setattr(layer_mod, "build_parallel_attention_strategy", lambda **kw: object())
|
||||
monkeypatch.setattr(layer_mod, "NoParallelAttention", lambda: object())
|
||||
custom = torch.nn.Identity()
|
||||
layer = layer_mod.Attention(
|
||||
num_heads=2,
|
||||
head_size=64,
|
||||
softmax_scale=0.125,
|
||||
causal=False,
|
||||
custom_attention=custom,
|
||||
skip_sequence_parallel=True,
|
||||
)
|
||||
assert layer.attention is custom and layer._kv_cache_dtype is None
|
||||
|
||||
|
||||
def test_legacy_global_fp8_uses_implicit_bsnd(runtime):
|
||||
q = torch.randn(1, 17, 2, 64, dtype=torch.bfloat16)
|
||||
impl = flash(layout=None)
|
||||
assert impl.forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": "fp8"})) is q
|
||||
assert runtime.quant_attention.call_args.kwargs["layout"] == "BSND"
|
||||
assert runtime.quant_attention.call_args.kwargs["precision"] == "fp8"
|
||||
@@ -133,18 +133,17 @@ def test_invalid_ref2va_video_spans_fall_back_to_dense():
|
||||
assert make_impl()._resolve_plan(AttentionMetadata(extra={"max_seqlen_q": 12000}, video_layout=layout)) is None
|
||||
|
||||
|
||||
def test_validate_available_rejects_legacy_mindiesd(monkeypatch):
|
||||
def test_validate_available_accepts_single_video_mindiesd(monkeypatch):
|
||||
mindiesd = types.ModuleType("mindiesd")
|
||||
|
||||
def sparse_attention(query, key, value, **kwargs):
|
||||
return query
|
||||
|
||||
mindiesd.sparse_attention = sparse_attention
|
||||
monkeypatch.setattr(mindiesd, "sparse_attention", sparse_attention, raising=False)
|
||||
monkeypatch.setitem(sys.modules, "mindiesd", mindiesd)
|
||||
monkeypatch.setattr("importlib.util.find_spec", lambda _: object())
|
||||
|
||||
with pytest.raises(ValueError, match="video_spans"):
|
||||
RainFusionAttentionBackend.validate_available()
|
||||
RainFusionAttentionBackend.validate_available()
|
||||
|
||||
|
||||
def test_validate_available_accepts_new_mindiesd(monkeypatch):
|
||||
@@ -153,13 +152,25 @@ def test_validate_available_accepts_new_mindiesd(monkeypatch):
|
||||
def sparse_attention(query, key, value, *, video_spans=None, **kwargs):
|
||||
return query
|
||||
|
||||
mindiesd.sparse_attention = sparse_attention
|
||||
monkeypatch.setattr(mindiesd, "sparse_attention", sparse_attention, raising=False)
|
||||
monkeypatch.setitem(sys.modules, "mindiesd", mindiesd)
|
||||
monkeypatch.setattr("importlib.util.find_spec", lambda _: object())
|
||||
|
||||
RainFusionAttentionBackend.validate_available()
|
||||
|
||||
|
||||
def test_prefix_kv_slicing_contract_is_pinned():
|
||||
# Regression pin for the #5543 packed-padding crash: reverting the flag
|
||||
# would pass CI while reintroducing the ValueError for non-64-aligned
|
||||
# requests. Both halves of the contract must hold together — the impl
|
||||
# trims [real, pad] packed tensors to the valid prefix itself (so it may
|
||||
# advertise prefix slicing) and it never reads attn_mask (so it must keep
|
||||
# refusing one), which is what lets the model skip materializing the
|
||||
# padding mask that _assert_metadata_compatible would reject.
|
||||
assert RainFusionAttentionBackend.supports_prefix_kv_slicing is True
|
||||
assert RainFusionAttentionBackend.supports_attention_mask() is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("grid", [(4, 24, 40), (1, 24, 40)])
|
||||
def test_short_video_stays_dense(grid):
|
||||
assert make_impl()._resolve_plan(make_metadata(grid)) is None
|
||||
@@ -300,22 +311,21 @@ def _fake_mindiesd_module():
|
||||
import types
|
||||
|
||||
fake = types.ModuleType("mindiesd")
|
||||
fake.sparse_attention = lambda *args, **kwargs: None
|
||||
setattr(fake, "sparse_attention", lambda *args, **kwargs: None)
|
||||
return fake
|
||||
|
||||
|
||||
def test_precision_non_bf16_requires_mindiesd_support():
|
||||
"""precision != bf16 against a mindiesd lacking the kwarg must raise RuntimeError."""
|
||||
"""Unsupported sparse precision must fail before any operator executes."""
|
||||
import sys
|
||||
|
||||
impl = make_impl(precision="mix")
|
||||
sys.modules["mindiesd"] = _fake_mindiesd_module()
|
||||
try:
|
||||
with mock.patch.dict(sys.modules, {"mindiesd": _fake_mindiesd_module()}):
|
||||
with mock.patch.object(rainfusion_attn, "_mindiesd_supports_precision", return_value=False):
|
||||
with pytest.raises(RuntimeError, match="requires MindIE-SD"):
|
||||
impl._forward_sparse_npu(None, None, None, None)
|
||||
finally:
|
||||
sys.modules.pop("mindiesd", None)
|
||||
with pytest.raises(ValueError, match="explicitly support precision"):
|
||||
impl._forward_sparse_npu(
|
||||
None, None, None, RainFusionPlan(prefix_len=0, used_len=8, latent_shape=[2, 2, 2])
|
||||
)
|
||||
|
||||
|
||||
def test_precision_non_bf16_passes_gate_when_supported():
|
||||
@@ -323,8 +333,7 @@ def test_precision_non_bf16_passes_gate_when_supported():
|
||||
import sys
|
||||
|
||||
impl = make_impl(precision="mix")
|
||||
sys.modules["mindiesd"] = _fake_mindiesd_module()
|
||||
try:
|
||||
with mock.patch.dict(sys.modules, {"mindiesd": _fake_mindiesd_module()}):
|
||||
with mock.patch.object(rainfusion_attn, "_mindiesd_supports_precision", return_value=True):
|
||||
# q/k/v shapes: [B, S, N, D]; plan geometry must match S.
|
||||
q = torch.randn(1, 8, 4, 128)
|
||||
@@ -332,5 +341,3 @@ def test_precision_non_bf16_passes_gate_when_supported():
|
||||
# The gate must pass; the fake mindiesd returns None so no crash.
|
||||
out = impl._forward_sparse_npu(q, q, q, plan)
|
||||
assert out is None
|
||||
finally:
|
||||
sys.modules.pop("mindiesd", None)
|
||||
@@ -0,0 +1,221 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.attention.backends import ring_flash_attn, ring_pytorch_attn
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.backends.ring.ring_selector import AttnType
|
||||
from vllm_omni.diffusion.attention.backends.ring.ring_utils import ring_kv_block_valid_length
|
||||
from vllm_omni.diffusion.attention.parallel import ring as ring_parallel
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
class _SingleRankComm:
|
||||
rank = 0
|
||||
world_size = 1
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _FakeSequenceParallelGroup:
|
||||
ring_group: object
|
||||
|
||||
|
||||
def _fake_attention_result(q, _k, _v, **_kwargs):
|
||||
return torch.zeros_like(q), torch.zeros(q.shape[0], q.shape[2], q.shape[1])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("block_rank", "expected"), [(0, 4), (1, 1), (2, 0)])
|
||||
def test_global_prefix_is_partitioned_across_ring_blocks(block_rank, expected):
|
||||
assert ring_kv_block_valid_length(5, 4, block_rank, 3) == expected
|
||||
|
||||
|
||||
def test_absent_prefix_length_keeps_the_full_ring_block():
|
||||
assert ring_kv_block_valid_length(None, 4, 2, 3) == 4
|
||||
|
||||
|
||||
def test_pytorch_ring_trims_padding_from_each_kv_block(monkeypatch):
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def fake_attention(q, k, v, **kwargs):
|
||||
captured.update(k=k, v=v, **kwargs)
|
||||
return _fake_attention_result(q, k, v, **kwargs)
|
||||
|
||||
monkeypatch.setattr(ring_pytorch_attn, "RingComm", lambda _group: _SingleRankComm())
|
||||
monkeypatch.setattr(ring_pytorch_attn, "pytorch_attn_forward", fake_attention)
|
||||
qkv = torch.randn(1, 4, 2, 8)
|
||||
|
||||
output = ring_pytorch_attn.ring_pytorch_attn_func(
|
||||
qkv,
|
||||
qkv,
|
||||
qkv,
|
||||
group=object(),
|
||||
valid_kv_length=3,
|
||||
)
|
||||
|
||||
assert output.shape == qkv.shape
|
||||
assert captured["k"].shape[1] == 3
|
||||
assert captured["v"].shape[1] == 3
|
||||
|
||||
|
||||
def test_pytorch_ring_trims_circulated_blocks_by_source_rank(monkeypatch):
|
||||
observed_lengths = []
|
||||
# Each ring step sends K and V separately.
|
||||
blocks = [torch.randn(1, 4, 2, 8) for _ in range(4)]
|
||||
|
||||
class FakeRingComm:
|
||||
rank = 2
|
||||
world_size = 3
|
||||
|
||||
def __init__(self, _group):
|
||||
self._next_block = 0
|
||||
|
||||
def send_recv(self, _tensor):
|
||||
block = blocks[self._next_block]
|
||||
self._next_block += 1
|
||||
return block
|
||||
|
||||
def commit(self):
|
||||
pass
|
||||
|
||||
def wait(self):
|
||||
pass
|
||||
|
||||
def fake_attention(q, k, v, **kwargs):
|
||||
observed_lengths.append(k.shape[1])
|
||||
return _fake_attention_result(q, k, v, **kwargs)
|
||||
|
||||
monkeypatch.setattr(ring_pytorch_attn, "RingComm", FakeRingComm)
|
||||
monkeypatch.setattr(ring_pytorch_attn, "pytorch_attn_forward", fake_attention)
|
||||
qkv = torch.randn(1, 4, 2, 8)
|
||||
|
||||
ring_pytorch_attn.ring_pytorch_attn_func(
|
||||
qkv,
|
||||
qkv,
|
||||
qkv,
|
||||
group=object(),
|
||||
valid_kv_length=5,
|
||||
)
|
||||
|
||||
# Rank 2 starts with global block 2, then receives blocks 1 and 0.
|
||||
# The first block is empty and therefore does not launch attention.
|
||||
assert observed_lengths == [1, 4]
|
||||
|
||||
|
||||
def test_flash_ring_trims_padding_from_each_kv_block(monkeypatch):
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def fake_attention(q, k, v, **kwargs):
|
||||
captured.update(k=k, v=v, **kwargs)
|
||||
return _fake_attention_result(q, k, v, **kwargs)
|
||||
|
||||
monkeypatch.setattr(ring_flash_attn, "RingComm", lambda _group: _SingleRankComm())
|
||||
monkeypatch.setattr(ring_flash_attn, "select_flash_attn_impl", lambda *_args, **_kwargs: fake_attention)
|
||||
qkv = torch.randn(1, 4, 2, 8)
|
||||
|
||||
output, _ = ring_flash_attn.ring_flash_attn_forward(
|
||||
object(),
|
||||
qkv,
|
||||
qkv,
|
||||
qkv,
|
||||
softmax_scale=0.25,
|
||||
causal=False,
|
||||
attn_type=AttnType.FA,
|
||||
valid_kv_length=3,
|
||||
)
|
||||
|
||||
assert output.shape == qkv.shape
|
||||
assert captured["k"].shape[1] == 3
|
||||
assert captured["v"].shape[1] == 3
|
||||
|
||||
|
||||
def test_ring_dispatch_forwards_global_valid_kv_length(monkeypatch):
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def fake_ring_attention(*_args, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return "output"
|
||||
|
||||
monkeypatch.setattr(ring_pytorch_attn, "ring_pytorch_attn_func", fake_ring_attention)
|
||||
strategy = ring_parallel.RingParallelAttention(
|
||||
_FakeSequenceParallelGroup(ring_group=object()),
|
||||
attn_backend_pref="sdpa",
|
||||
)
|
||||
metadata = AttentionMetadata(extra={"valid_kv_length": 3})
|
||||
query = torch.randn(1, 4, 2, 8)
|
||||
|
||||
assert strategy.run_attention(query, query, query, metadata) == "output"
|
||||
assert captured["valid_kv_length"] == 3
|
||||
|
||||
|
||||
def test_pytorch_ring_rejects_causal_with_a_valid_prefix(monkeypatch):
|
||||
"""Trimming a K/V block under causal=True would silently shift the diagonal.
|
||||
|
||||
The query block keeps the full padded length while the circulated K/V block
|
||||
is trimmed, and a causal mask over a rectangular tile is bottom-right
|
||||
aligned, so the combination must be refused rather than computed.
|
||||
"""
|
||||
monkeypatch.setattr(ring_pytorch_attn, "RingComm", lambda _group: _SingleRankComm())
|
||||
monkeypatch.setattr(ring_pytorch_attn, "pytorch_attn_forward", _fake_attention_result)
|
||||
qkv = torch.randn(1, 4, 2, 8)
|
||||
|
||||
with pytest.raises(ValueError, match="valid_kv_length is not supported with causal=True"):
|
||||
ring_pytorch_attn.ring_pytorch_attn_func(
|
||||
qkv,
|
||||
qkv,
|
||||
qkv,
|
||||
causal=True,
|
||||
group=object(),
|
||||
valid_kv_length=3,
|
||||
)
|
||||
|
||||
|
||||
def test_flash_ring_rejects_causal_with_a_valid_prefix(monkeypatch):
|
||||
"""Same refusal on the Flash-Attention ring path."""
|
||||
monkeypatch.setattr(ring_flash_attn, "RingComm", lambda _group: _SingleRankComm())
|
||||
monkeypatch.setattr(
|
||||
ring_flash_attn,
|
||||
"select_flash_attn_impl",
|
||||
lambda *_args, **_kwargs: _fake_attention_result,
|
||||
)
|
||||
qkv = torch.randn(1, 4, 2, 8)
|
||||
|
||||
with pytest.raises(ValueError, match="valid_kv_length is not supported with causal=True"):
|
||||
ring_flash_attn.ring_flash_attn_forward(
|
||||
object(),
|
||||
qkv,
|
||||
qkv,
|
||||
qkv,
|
||||
softmax_scale=0.25,
|
||||
causal=True,
|
||||
attn_type=AttnType.FA,
|
||||
valid_kv_length=3,
|
||||
)
|
||||
|
||||
|
||||
def test_ring_paths_still_accept_causal_without_a_valid_prefix(monkeypatch):
|
||||
"""The guard is scoped to the padded case: plain causal Ring is unchanged."""
|
||||
monkeypatch.setattr(ring_flash_attn, "RingComm", lambda _group: _SingleRankComm())
|
||||
monkeypatch.setattr(
|
||||
ring_flash_attn,
|
||||
"select_flash_attn_impl",
|
||||
lambda *_args, **_kwargs: _fake_attention_result,
|
||||
)
|
||||
qkv = torch.randn(1, 4, 2, 8)
|
||||
|
||||
output, _ = ring_flash_attn.ring_flash_attn_forward(
|
||||
object(),
|
||||
qkv,
|
||||
qkv,
|
||||
qkv,
|
||||
softmax_scale=0.25,
|
||||
causal=True,
|
||||
attn_type=AttnType.FA,
|
||||
)
|
||||
|
||||
assert output.shape == qkv.shape
|
||||
Reference in new issue
Block a user