Merge remote-tracking branch 'origin/main' into qwen-image-2.1

# Conflicts:
#	vllm_omni/diffusion/executor/multiproc_executor.py
#	vllm_omni/diffusion/offloader/sequential_backend.py
This commit is contained in:
Isotr0py committed 2026-09-21 07:32:04 +00:00
commit 9174554e40
1332 files changed
+160051 -60504

No files matched your search

@@ -10,6 +10,7 @@ Tests cover:
- AttentionMetadata.extra field
"""
from dataclasses import replace
from types import SimpleNamespace
from unittest.mock import Mock
@@ -17,7 +18,12 @@ import pytest
import torch
import vllm_omni.diffusion.attention.layer as layer_mod
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
import vllm_omni.diffusion.attention.selector as selector_mod
from vllm_omni.diffusion.attention.backends.abstract import (
AttentionMetadata,
PackedPaddingMetadata,
QueryRange,
)
from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionBackend
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.config import (
@@ -477,6 +483,7 @@ class TestAttentionInitUsesCurrentDiffusionConfig:
@pytest.mark.parametrize("mindiesd_available", [False, True])
def test_paged_npu_memory_profile_uses_sdpa_only_without_mindiesd(self, monkeypatch, mindiesd_available):
attention = Attention.__new__(Attention)
attention.allow_fp32_fallback = False
attention._scheduler_paged_kv = True
attention.paged_kv_cache_role = "primary"
attention.backend_pref = "FLASH_ATTN"
@@ -519,6 +526,7 @@ class TestAttentionInitUsesCurrentDiffusionConfig:
@pytest.mark.parametrize("fa4_available", [False, True])
def test_paged_cuda_memory_profile_uses_sdpa_without_dense_fa4(self, monkeypatch, fa4_available):
attention = Attention.__new__(Attention)
attention.allow_fp32_fallback = False
attention._scheduler_paged_kv = True
attention.paged_kv_cache_role = "primary"
attention.backend_pref = "FLASH_ATTN"
@@ -561,6 +569,7 @@ class TestAttentionInitUsesCurrentDiffusionConfig:
def test_dense_flash_import_error_is_not_reclassified(self, monkeypatch):
attention = Attention.__new__(Attention)
attention.allow_fp32_fallback = False
attention._scheduler_paged_kv = False
attention.paged_kv_cache_role = "primary"
attention.backend_pref = "FLASH_ATTN"
@@ -900,6 +909,7 @@ class TestAttentionInitUsesCurrentDiffusionConfig:
return sentinel
fake_attention = SimpleNamespace(
allow_fp32_fallback=False,
attention=SimpleNamespace(
forward=_selected_forward,
),
@@ -938,6 +948,164 @@ class TestAttentionInitUsesCurrentDiffusionConfig:
Attention._run_ring_attention(fake_attention, query, query, query, metadata)
class TestOptInFloat32Fallback:
@staticmethod
def _make_attention(monkeypatch, *, backend=None, allow=True, kv_cache_dtype=None):
events, state = [], {}
context = object()
class _SelectedImpl:
def __init__(self, **kwargs):
self.kwargs = kwargs
@staticmethod
def supports_kv_cache_dtype(*args):
return True
def forward(self, query, key, value, metadata):
events.append("selected")
state["kernel_call"] = (query, key, value, metadata)
state["kernel_output"] = query + 4
return state["kernel_output"]
class _FallbackImpl(_SelectedImpl):
def forward(self, query, key, value, metadata):
output = super().forward(query, key, value, metadata)
events[-1] = "fallback"
return output
class _FlashBackend(FlashAttentionBackend):
@staticmethod
def get_impl_cls():
return _SelectedImpl
class _SDPABackend(layer_mod.SDPABackend):
@staticmethod
def get_impl_cls():
return _SelectedImpl
class _Strategy:
def pre_attention(self, query, key, value, metadata):
events.append("pre")
state["prepared"] = (
query + 1,
key + 2,
value + 3,
replace(metadata) if metadata is not None else None,
)
return (*state["prepared"], context)
def post_attention(self, output, ctx):
events.append("post")
assert output is state["kernel_output"]
assert ctx is context
return output + 7
# Keep real role/config resolution and construction; replace hardware
# lookup and kernels. These are CPU dispatch tests, not SP collectives.
monkeypatch.setattr(
selector_mod,
"_cached_get_backend_cls",
lambda name, *args, **kwargs: _SDPABackend if name == "TORCH_SDPA" else _FlashBackend,
)
monkeypatch.setattr(layer_mod.SDPABackend, "get_impl_cls", staticmethod(lambda: _FallbackImpl))
monkeypatch.setattr(layer_mod, "build_parallel_attention_strategy", lambda **kwargs: _Strategy())
monkeypatch.setattr(layer_mod, "is_forward_context_available", lambda: False)
config = OmniDiffusionConfig(
diffusion_attention_config={"default": backend} if backend else {},
diffusion_kv_cache_dtype=kv_cache_dtype,
)
with set_current_diffusion_config(config):
attention = Attention(
num_heads=4,
num_kv_heads=2,
head_size=8,
causal=False,
softmax_scale=0.25,
**({"allow_fp32_fallback": True} if allow else {}),
)
return attention, events, state
@pytest.mark.parametrize(
"backend,allow,dtype,is_cuda,expected",
[
pytest.param(None, True, torch.float32, True, "fallback", id="auto-fp32"),
pytest.param(None, False, torch.float32, True, "selected", id="default-opt-out"),
pytest.param("FLASH_ATTN", True, torch.float32, True, "selected", id="explicit-flash"),
pytest.param("TORCH_SDPA", True, torch.float32, True, "selected", id="explicit-sdpa"),
pytest.param(None, True, torch.bfloat16, True, "selected", id="bf16"),
pytest.param(None, True, torch.float32, False, "selected", id="cpu-fp32"),
],
)
def test_public_forward_preserves_strategy_and_backend_choice(
self, monkeypatch, backend, allow, dtype, is_cuda, expected
):
attention, events, state = self._make_attention(monkeypatch, backend=backend, allow=allow)
# Only the dispatch predicate is simulated; every tensor stays on CPU.
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: is_cuda))
query = torch.ones(1, 2, 4, 8, dtype=dtype)
key = torch.full((1, 2, 2, 8), 2, dtype=dtype)
value = torch.full_like(key, 3)
metadata = AttentionMetadata(attn_mask=torch.tensor([[True, False]]))
output = attention(query, key, value, metadata)
assert events == ["pre", expected, "post"]
assert all(got is prepared for got, prepared in zip(state["kernel_call"], state["prepared"]))
assert state["kernel_call"][3] is not metadata
assert state["kernel_call"][3].attn_mask is metadata.attn_mask
torch.testing.assert_close(output, query + 12)
assert attention.backend_explicit is (backend is not None)
impl = attention.sdpa_fallback if expected == "fallback" else attention.attention
assert impl.kwargs["num_kv_heads"] == 2
assert impl.kwargs["softmax_scale"] == 0.25
def test_explicit_backend_failure_is_not_retried_with_sdpa(self, monkeypatch):
attention, events, _ = self._make_attention(monkeypatch, backend="FLASH_ATTN")
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
selected = Mock(side_effect=RuntimeError("selected backend rejects this dtype"))
monkeypatch.setattr(attention.attention, "forward", selected)
query = torch.ones(1, 2, 4, 8)
with pytest.raises(RuntimeError, match="selected backend rejects this dtype"):
attention(query, query, query)
selected.assert_called_once()
assert events == ["pre"]
@pytest.mark.parametrize("contract", ["packed", "packed-padding", "piecewise", "quantized-kv"])
def test_backend_metadata_is_not_silently_sent_to_dense_sdpa(self, monkeypatch, contract):
quant_dtype = "fp8" if contract == "quantized-kv" else None
attention, events, state = self._make_attention(monkeypatch, kv_cache_dtype=quant_dtype)
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
lengths = torch.tensor([0, 1, 2], dtype=torch.int32)
metadata = AttentionMetadata()
if contract == "packed":
metadata.extra = {
"cu_seqlens_q": lengths,
"cu_seqlens_k": lengths,
"max_seqlen_q": 1,
"max_seqlen_k": 1,
}
elif contract == "packed-padding":
metadata.packed_padding = PackedPaddingMetadata(1, 1, lengths[:2], lengths[:2])
elif contract == "piecewise":
metadata.full_attn_spans = [[(0, 1)]]
metadata.query_ranges = (QueryRange(0, 2, 0),)
query = torch.ones(1, 2, 4, 8)
attention(query, query, query, metadata)
assert events == ["pre", "selected", "post"]
forwarded = state["kernel_call"][3]
if quant_dtype:
# Quantization is injected by the public forward from config.
assert forwarded.extra == {"kv_cache_dtype": "fp8"}
assert metadata.extra == {}
else:
assert forwarded is state["prepared"][3]
class TestDiffusionKvCacheQuantization:
@staticmethod
def _install_attention_init_stubs(monkeypatch, *, supports_kv_cache_dtype=True):
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import sys
import types
import pytest
import torch
from torch._subclasses.fake_tensor import FakeTensorMode
from vllm_omni.diffusion.attention.ops.block_sparse import (
build_prefix_dense_block_map,
fastvideo_block_sparse_attn_bshd,
mean_pool_tiles,
)
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
@pytest.mark.parametrize("prefix_blocks,topk", [(0, 1), (2, 0), (2, 1), (2, 3), (2, 5)])
def test_block_map_keeps_prefix_dense_and_selects_non_prefix_keys(prefix_blocks, topk):
sparse_blocks = 3
blocks = prefix_blocks + sparse_blocks
scores = torch.arange(2 * blocks * blocks, dtype=torch.float32).reshape(1, 2, blocks, blocks)
block_map = build_prefix_dense_block_map(scores, prefix_blocks, sparse_blocks, topk)
expected = torch.zeros_like(scores, dtype=torch.bool)
# Scores increase with key index, so top-k always selects the last keys.
for query_block in range(blocks):
for key_block in range(blocks):
if (
query_block < prefix_blocks
or key_block < prefix_blocks
or key_block >= blocks - min(topk, sparse_blocks)
):
expected[..., query_block, key_block] = True
assert torch.equal(block_map, expected)
@pytest.mark.parametrize("block_size", [4, 32, 64])
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
def test_pooling_uses_valid_tile_sizes_and_fp32_accumulation(block_size, dtype):
sizes = torch.tensor([block_size, block_size - 1, 0], dtype=torch.int32)
tiles = torch.arange(2 * 3 * block_size * 2 * 3).reshape(2, 3, block_size, 2, 3).to(dtype)
tiles[:, 1, -1] = 0
tiles[:, 2] = 0
expected = torch.stack(
[tiles[:, 0].float().mean(dim=1), tiles[:, 1, :-1].float().mean(dim=1), torch.zeros(2, 2, 3)], dim=2
)
result = mean_pool_tiles(tiles.flatten(1, 2), sizes, block_size)
assert result.dtype == torch.float32
torch.testing.assert_close(result, expected, rtol=0, atol=0)
def test_provider_wrapper_restores_transport_rows_and_bshd_layout(monkeypatch):
monkeypatch.delenv("FASTVIDEO_VSA_SM100A", raising=False)
calls = []
def block_sparse_attn(q, k, v, block_map, sizes):
calls.append((q.shape, block_map.shape, sizes.tolist()))
assert q.is_contiguous() and k.is_contiguous() and v.is_contiguous()
assert sizes.dtype == torch.int32
return q + k + v, None
module = types.ModuleType("fastvideo_kernel.block_sparse_attn")
setattr(module, "block_sparse_attn", block_sparse_attn)
monkeypatch.setitem(sys.modules, "fastvideo_kernel", types.ModuleType("fastvideo_kernel"))
monkeypatch.setitem(sys.modules, "fastvideo_kernel.block_sparse_attn", module)
query = torch.randn(1, 4 * 64, 2, 8)
block_map = torch.ones(1, 2, 4, 4, dtype=torch.bool)
sizes = torch.tensor([64, 17, 64, 0])
output = fastvideo_block_sparse_attn_bshd(query, query, query, block_map, sizes, 3)
expected = torch.zeros_like(query)
expected[:, : 3 * 64] = query[:, : 3 * 64] * 3
torch.testing.assert_close(output, expected)
assert calls == [(torch.Size([1, 2, 3 * 64, 8]), torch.Size([1, 2, 3, 3]), [64, 17, 64])]
def test_custom_op_fake_preserves_metadata_without_loading_provider(monkeypatch):
monkeypatch.setitem(sys.modules, "fastvideo_kernel", None)
with FakeTensorMode():
query = torch.empty(1, 4 * 64, 2, 8, dtype=torch.bfloat16)
block_map = torch.empty(1, 2, 4, 4, dtype=torch.bool)
sizes = torch.empty(4, dtype=torch.int32)
output = fastvideo_block_sparse_attn_bshd(query, query, query, block_map, sizes, 3)
assert output.shape == query.shape
assert output.dtype == query.dtype
assert output.device == query.device
@pytest.mark.parametrize("error_type", [ImportError, RuntimeError, torch.AcceleratorError])
def test_native_provider_failure_only_recovers_without_accelerator_fault(monkeypatch, mocker, error_type):
monkeypatch.setenv("FASTVIDEO_VSA_SM100A", "1")
error = error_type("native provider failure")
native = mocker.Mock()
native.is_supported.return_value = True
native.block_sparse_attn_sm100a.side_effect = error
provider = types.ModuleType("fastvideo_kernel")
setattr(provider, "block_sparse_attn_sm100a", native)
index = types.ModuleType("fastvideo_kernel.triton_kernels.index")
setattr(
index,
"map_to_index",
mocker.Mock(return_value=(torch.zeros(1, dtype=torch.int32), torch.ones(1, dtype=torch.int32))),
)
triton = types.ModuleType("fastvideo_kernel.block_sparse_attn")
fallback = mocker.Mock(side_effect=lambda q, *args: (q.clone(), None))
setattr(triton, "block_sparse_attn", fallback)
monkeypatch.setitem(sys.modules, "fastvideo_kernel", provider)
monkeypatch.setitem(
sys.modules, "fastvideo_kernel.triton_kernels", types.ModuleType("fastvideo_kernel.triton_kernels")
)
monkeypatch.setitem(sys.modules, "fastvideo_kernel.triton_kernels.index", index)
monkeypatch.setitem(sys.modules, "fastvideo_kernel.block_sparse_attn", triton)
query = torch.arange(128 * 2 * 8, dtype=torch.float32).reshape(1, 128, 2, 8)
block_map = torch.ones(1, 2, 2, 2, dtype=torch.bool)
sizes = torch.tensor([64, 64], dtype=torch.int32)
if error_type is torch.AcceleratorError:
with pytest.raises(torch.AcceleratorError, match="native provider failure") as raised:
fastvideo_block_sparse_attn_bshd(query, query, query, block_map, sizes, 2)
assert raised.value is error
fallback.assert_not_called()
else:
output = fastvideo_block_sparse_attn_bshd(query, query, query, block_map, sizes, 2)
torch.testing.assert_close(output, query, rtol=0, atol=0)
fallback.assert_called_once()
+3 -161
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import math
import sys
import types
from typing import Any
@@ -12,14 +11,10 @@ import torch
from vllm_omni.diffusion.attention.backends.abstract import (
AttentionMetadata,
PackedPaddingMetadata,
VideoTokenLayout,
VideoTokenSpan,
)
from vllm_omni.diffusion.attention.backends.fastvideo_vsa import (
FastVideoVSABackend,
FastVideoVSAImpl,
_build_h3_block_map,
_get_h3_tile_metadata,
)
from vllm_omni.diffusion.attention.backends.registry import (
DiffusionAttentionBackendEnum,
@@ -218,144 +213,8 @@ def test_fastvideo_vsa_allows_topk_equal_to_num_blocks():
assert too_many._fallback_reason(query, query, query, metadata) == "topk 3 > num_blocks 2"
def _h3_metadata(prefix_segments, video_shape, *, gate=None, packed_padding=None):
extra: dict[str, Any] = {
"vsa_h3_prefix_segments": prefix_segments,
}
if gate is not None:
extra["gate_compress"] = gate
return AttentionMetadata(
packed_padding=packed_padding,
video_layout=VideoTokenLayout(
used_len=sum(prefix_segments) + math.prod(video_shape),
video_spans=(VideoTokenSpan(start=sum(prefix_segments), latent_grid=video_shape, role="target"),),
),
extra=extra,
)
def _fake_block_sparse_kernel(monkeypatch, calls):
"""Stand in for fastvideo_kernel.block_sparse_attn with a mask-obeying SDPA."""
def block_sparse_attn(q, k, v, block_map, variable_block_sizes):
calls["block_map"] = block_map.clone()
calls["variable_block_sizes"] = variable_block_sizes.tolist()
calls["q_shape"] = tuple(q.shape)
blocks = block_map.shape[-1]
assert blocks * 64 == q.shape[2]
# Expand the per-block map to a per-token mask so the stand-in honours
# the sparsity contract the real kernel implements, and drop the rows
# each partial block pads with, which the real kernel skips through
# variable_block_sizes.
token_mask = block_map.repeat_interleave(64, dim=-1).repeat_interleave(64, dim=-2)
within = torch.arange(64, device=q.device)[None, :] < variable_block_sizes[:, None]
token_mask = token_mask & within.reshape(1, 1, 1, -1)
out = torch.nn.functional.scaled_dot_product_attention(q.float(), k.float(), v.float(), attn_mask=token_mask)
return out.to(q.dtype), None
module = types.ModuleType("fastvideo_kernel.block_sparse_attn")
setattr(module, "block_sparse_attn", block_sparse_attn)
monkeypatch.setitem(sys.modules, "fastvideo_kernel", types.ModuleType("fastvideo_kernel"))
monkeypatch.setitem(sys.modules, "fastvideo_kernel.block_sparse_attn", module)
def _h3_impl(**backend_kwargs):
kwargs = {"topk": 1, "min_seq_len": 1, "disable_when_sp_active": False}
kwargs.update(backend_kwargs)
return FastVideoVSAImpl(num_heads=2, head_size=8, softmax_scale=8**-0.5, backend_kwargs=kwargs)
def test_h3_forward_tiles_untiles_and_restores_the_original_row_order(monkeypatch):
calls: dict[str, Any] = {}
_fake_block_sparse_kernel(monkeypatch, calls)
impl = _h3_impl(topk=64)
prefix_segments, video_shape = (5, 70), (2, 4, 4)
seq_len = sum(prefix_segments) + 2 * 4 * 4
torch.manual_seed(0)
query = torch.randn(1, seq_len, 2, 8, dtype=torch.bfloat16)
output = impl.forward_cuda(query, query, query, _h3_metadata(prefix_segments, video_shape))
assert output.shape == query.shape
# topk covers every video block, so the block map is dense and the tiled
# kernel must reproduce plain attention over the same rows.
reference = torch.nn.functional.scaled_dot_product_attention(
*(x.float().transpose(1, 2) for x in (query, query, query))
).transpose(1, 2)
torch.testing.assert_close(output.float(), reference, atol=2e-2, rtol=2e-2)
# 5 + 70 splits into segment-pure 5 | 64 | 6, then one (2, 4, 4) video tile.
assert calls["variable_block_sizes"] == [5, 64, 6, 32]
def test_h3_forward_keeps_prefix_dense_and_selects_top_k_video_tiles(monkeypatch):
calls: dict[str, Any] = {}
_fake_block_sparse_kernel(monkeypatch, calls)
impl = _h3_impl(topk=1)
prefix_segments, video_shape = (5,), (8, 4, 4)
seq_len = 5 + 8 * 4 * 4
torch.manual_seed(0)
query = torch.randn(1, seq_len, 2, 8, dtype=torch.bfloat16)
impl.forward_cuda(query, query, query, _h3_metadata(prefix_segments, video_shape))
# 1 prefix block + 2 video blocks. The odd count is padded to an even one
# for the paired-CTA kernel contract, and the Triton route slices that
# transport-only partner off again before launching.
block_map = calls["block_map"]
assert block_map.shape[-1] == 3
assert calls["variable_block_sizes"] == [5, 64, 64]
assert block_map[:, :, :1, :].all(), "prefix queries stay dense"
assert block_map[..., :1].all(), "prefix keys are exempt from selection"
assert (block_map[:, :, 1:, 1:].sum(dim=-1) == 1).all(), "one video tile per video query"
# The same odd geometry must still untile back onto the original rows.
dense = _h3_impl(topk=64).forward_cuda(query, query, query, _h3_metadata(prefix_segments, video_shape))
reference = torch.nn.functional.scaled_dot_product_attention(
*(x.float().transpose(1, 2) for x in (query, query, query))
).transpose(1, 2)
torch.testing.assert_close(dense.float(), reference, atol=2e-2, rtol=2e-2)
def test_h3_forward_applies_the_learned_compression_gate(monkeypatch):
_fake_block_sparse_kernel(monkeypatch, {})
impl = _h3_impl(topk=64)
prefix_segments, video_shape = (5,), (2, 4, 4)
seq_len = 5 + 2 * 4 * 4
torch.manual_seed(0)
query = torch.randn(1, seq_len, 2, 8, dtype=torch.bfloat16)
without_gate = impl.forward_cuda(query, query, query, _h3_metadata(prefix_segments, video_shape))
zero_gate = torch.zeros_like(query)
with_zero_gate = impl.forward_cuda(query, query, query, _h3_metadata(prefix_segments, video_shape, gate=zero_gate))
live_gate = torch.full_like(query, 0.5)
with_live_gate = impl.forward_cuda(query, query, query, _h3_metadata(prefix_segments, video_shape, gate=live_gate))
torch.testing.assert_close(with_zero_gate.float(), without_gate.float())
assert not torch.allclose(with_live_gate.float(), without_gate.float())
def test_h3_forward_restores_rows_the_packed_padding_excludes(monkeypatch):
_fake_block_sparse_kernel(monkeypatch, {})
impl = _h3_impl(topk=64)
prefix_segments, video_shape = (5,), (2, 4, 4)
valid = 5 + 2 * 4 * 4
torch.manual_seed(0)
query = torch.randn(1, valid + 24, 2, 8, dtype=torch.bfloat16)
padding = PackedPaddingMetadata(
q_length=valid,
kv_length=valid,
cu_seqlens_q=torch.tensor([0, valid], dtype=torch.int32),
cu_seqlens_k=torch.tensor([0, valid], dtype=torch.int32),
)
output = impl.forward_cuda(query, query, query, _h3_metadata(prefix_segments, video_shape, packed_padding=padding))
assert output.shape == query.shape
assert torch.count_nonzero(output[:, valid:]) == 0
def test_sdpa_fallback_never_attends_the_structural_padding(monkeypatch):
# The backend advertises supports_packed_mask_free, so MiniMax-H3 skips
# The backend advertises supports_packed_mask_free, so a model may skip
# building the padding mask. Every fallback out of the VSA path must honour
# that contract itself; SDPA reads attn_mask and nothing else.
impl = FastVideoVSAImpl(
@@ -374,6 +233,8 @@ def test_sdpa_fallback_never_attends_the_structural_padding(monkeypatch):
cu_seqlens_k=torch.tensor([0, valid], dtype=torch.int32),
)
monkeypatch.setattr(impl.sdpa_fallback, "forward", impl.sdpa_fallback.forward_cuda)
baseline = impl.forward_cuda(query, query, query, AttentionMetadata(packed_padding=padding, extra={}))
perturbed_input = query.clone()
perturbed_input[:, valid:] += 100.0
@@ -384,22 +245,3 @@ def test_sdpa_fallback_never_attends_the_structural_padding(monkeypatch):
assert baseline.shape == query.shape
torch.testing.assert_close(baseline[:, :valid], perturbed[:, :valid])
assert torch.count_nonzero(baseline[:, valid:]) == 0
def test_h3_geometry_keeps_prefix_segments_pure_and_tiles_video_3d():
partition, sizes, non_pad, untile, prefix_blocks, video_blocks = _get_h3_tile_metadata(
(5, 70, 9), (5, 6, 6), torch.device("cpu")
)
assert sizes[:4].tolist() == [5, 64, 6, 9]
assert prefix_blocks == 4
assert video_blocks == 8
assert int(sizes.sum()) == 5 + 70 + 9 + 5 * 6 * 6
assert partition.numel() == non_pad.numel() == untile.numel() == int(sizes.sum())
def test_h3_block_map_makes_prefix_queries_dense_and_prefix_keys_exempt():
scores = torch.arange(1 * 2 * 5 * 5, dtype=torch.float32).reshape(1, 2, 5, 5)
block_map = _build_h3_block_map(scores, num_prefix_blocks=2, num_video_blocks=3, topk=1)
assert block_map[:, :, :2].all()
assert block_map[..., :2].all()
assert (block_map[:, :, 2:, 2:].sum(dim=-1) == 1).all()
+144 -9
View File
@@ -16,13 +16,14 @@ from unittest.mock import Mock
import pytest
import torch
from tests.helpers.mark import hardware_test
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionImpl
from vllm_omni.diffusion.attention.backends.sdpa import SDPAImpl
from vllm_omni.diffusion.attention.backends.utils import fa # noqa: E402
from vllm_omni.platforms import current_omni_platform
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
is_gpu = current_omni_platform.is_cuda_alike() or current_omni_platform.is_xpu()
HAS_FLASH_ATTN = fa.HAS_FLASH_ATTN
@@ -42,10 +43,12 @@ flash_attn_func = fa.flash_attn_func # noqa: N813
(9, "4", frozenset((2, 3, 4)), 4),
],
)
@pytest.mark.cpu
def test_versioned_flash_attention_selection(device_major, requested, supported, expected):
assert fa._choose_vllm_flash_attn_version(device_major, requested, supported) == expected
@pytest.mark.cpu
def test_versioned_flash_attention_selection_rejects_invalid_or_unavailable_version():
with pytest.raises(ValueError, match="must be 2, 3, or 4"):
fa._choose_vllm_flash_attn_version(9, "5", frozenset((2, 3, 4)))
@@ -92,7 +95,8 @@ def pad_tensor(tensor: torch.Tensor, target_seq_len: int, pad_value: float = 0.0
return torch.cat([tensor, padding], dim=1)
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires an accelerator")
def test_padding_equivalence():
"""
Case 1: Test that padded and unpadded inputs produce similar outputs.
@@ -185,7 +189,8 @@ def test_padding_equivalence():
print("✓ Case 1 PASSED: Padded and unpadded outputs are very close!")
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires an accelerator")
def test_fa_vs_sdpa():
"""
Case 2: Compare FlashAttention and SDPA backends with padding.
@@ -302,7 +307,8 @@ def test_fa_vs_sdpa():
print("✓ Case 2 PASSED: FA and SDPA outputs are very close!")
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires an accelerator")
def test_flash_attn_func_preferred_over_varlen():
"""Test flash_attn_func availability and basic forward call."""
if not HAS_FLASH_ATTN:
@@ -332,7 +338,8 @@ def test_flash_attn_func_preferred_over_varlen():
print("✓ flash_attn_func forward works correctly!")
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires CUDA or XPU")
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
@pytest.mark.skipif(not is_gpu, reason="FlashAttention requires an accelerator")
@pytest.mark.parametrize("k_len", [40, 256])
def test_cross_attn_key_padding_vs_sdpa(k_len):
"""
@@ -390,6 +397,7 @@ def test_cross_attn_key_padding_vs_sdpa(k_len):
print("✓ Case 3 PASSED: FA and SDPA cross-attention outputs are very close!")
@pytest.mark.cpu
def test_varlen_masked_routing_by_role(monkeypatch):
"""The unpad route is picked by role, not Q/K length equality; runs without a GPU."""
calls = []
@@ -424,6 +432,7 @@ def test_varlen_masked_routing_by_role(monkeypatch):
assert cu_seqlens_k.tolist() == [0, 2, 5]
@pytest.mark.cpu
def test_piecewise_flash_attn_uses_varlen_fallback(monkeypatch):
calls = []
@@ -454,6 +463,7 @@ def test_piecewise_flash_attn_uses_varlen_fallback(monkeypatch):
assert calls[0]["softmax_scale"] == 0.5
@pytest.mark.cpu
def test_packed_varlen_metadata_bypasses_mask_unpadding(monkeypatch):
calls = []
@@ -486,6 +496,7 @@ def test_packed_varlen_metadata_bypasses_mask_unpadding(monkeypatch):
assert calls[0][3]["max_seqlen_q"] == 6
@pytest.mark.cpu
def test_packed_varlen_metadata_must_be_complete(monkeypatch):
monkeypatch.setattr(fa, "HAS_FLASH_ATTN", True)
impl = FlashAttentionImpl(num_heads=2, head_size=4, softmax_scale=0.5, causal=False)
@@ -503,11 +514,12 @@ def test_packed_varlen_metadata_must_be_complete(monkeypatch):
# - boundary resolution in ``_resolve_packed_seq_npu`` (pure Python, no device
# sync, no MindIE-SD import);
# - env dispatch in ``forward_fa_npu`` (which op a packed forward takes);
# - native torch_npu dispatch for right-down causal attention;
# - the laser input pre-scaling math in ``_forward_prefix_kv_slice_npu``.
#
# They run on CPU without a real NPU by injecting a fake ``mindiesd`` module
# into ``sys.modules`` (same pattern as ``test_paged_attention`` /
# ``benchmarks/conftest``). No production code is exercised for real kernels.
# They run on CPU without a real NPU by injecting fake ``mindiesd`` and
# ``torch_npu`` modules into ``sys.modules`` (same pattern as
# ``test_paged_attention`` / ``benchmarks/conftest``). No real kernel runs.
def _npu_impl(causal: bool = False) -> FlashAttentionImpl:
@@ -529,9 +541,19 @@ def _fake_mindiesd(monkeypatch, *, attention_forward=None, attention_forward_var
return fake
def _fake_torch_npu(monkeypatch, *, npu_fusion_attention=None):
"""Install the minimal torch_npu surface used by causal NPU attention."""
fake = SimpleNamespace(
npu_fusion_attention=npu_fusion_attention or Mock(return_value=(torch.zeros(1), None, None, None, 0, 0, 0)),
)
monkeypatch.setitem(sys.modules, "torch_npu", fake)
return fake
# --- Test group A: boundary resolution (_resolve_packed_seq_npu) -------------
@pytest.mark.cpu
def test_resolve_packed_seq_accepts_real_plus_pad_contract():
impl = _npu_impl()
q = torch.randn(1, 8, 2, 4)
@@ -541,6 +563,7 @@ def test_resolve_packed_seq_accepts_real_plus_pad_contract():
assert impl._resolve_packed_seq_npu(q, q, extra) == ([5, 8], [5, 8])
@pytest.mark.cpu
def test_resolve_packed_seq_single_document_no_padding():
impl = _npu_impl()
q = torch.randn(1, 8, 2, 4)
@@ -564,6 +587,7 @@ def test_resolve_packed_seq_single_document_no_padding():
"pad_longer_than_real",
],
)
@pytest.mark.cpu
def test_resolve_packed_seq_rejects_malformed_metadata(case):
"""Any metadata outside the exact [real, pad] contract returns None so the
caller falls back to the masked path."""
@@ -620,6 +644,7 @@ def _npu_impl_with_mocked_paths() -> tuple[FlashAttentionImpl, torch.Tensor]:
return impl, sentinel
@pytest.mark.cpu
def test_npu_env_unset_uses_varlen(monkeypatch):
monkeypatch.delenv("MINDIE_SD_FA_TYPE", raising=False)
_fake_mindiesd(monkeypatch)
@@ -632,6 +657,7 @@ def test_npu_env_unset_uses_varlen(monkeypatch):
impl._forward_prefix_kv_slice_npu.assert_not_called()
@pytest.mark.cpu
def test_npu_env_laser_uses_slice(monkeypatch):
monkeypatch.setenv("MINDIE_SD_FA_TYPE", "ascend_laser_attention")
_fake_mindiesd(monkeypatch)
@@ -645,6 +671,7 @@ def test_npu_env_laser_uses_slice(monkeypatch):
@pytest.mark.parametrize("fa_type", ["prompt_flash_attn", "fused_attn_score", "ascend_laser_attenton", "garbage"])
@pytest.mark.cpu
def test_npu_env_non_laser_value_falls_back_to_varlen(monkeypatch, fa_type):
"""Locks in the CURRENT dispatch behavior (PR #5891).
@@ -665,6 +692,7 @@ def test_npu_env_non_laser_value_falls_back_to_varlen(monkeypatch, fa_type):
impl._forward_prefix_kv_slice_npu.assert_not_called()
@pytest.mark.cpu
def test_npu_varlen_opt_in_unset_takes_mask_path(monkeypatch):
"""Models that do not set ``npu_attn_varlen`` (e.g. Wan/Cosmos) keep using
the existing masked attention_forward path; the new branches are skipped."""
@@ -685,9 +713,115 @@ def test_npu_varlen_opt_in_unset_takes_mask_path(monkeypatch):
assert out is fake_forward.return_value
# --- Test group C: laser input pre-scaling in _forward_prefix_kv_slice_npu --
# --- Test group C: native NPU causal attention dispatch ----------------------
@pytest.mark.parametrize(
("query_length", "key_length", "expected_inner_precise"),
[(4, 4, 0), (2, 4, 0), (4, 2, 2)],
)
@pytest.mark.cpu
def test_npu_causal_uses_native_right_down_mode(
monkeypatch,
query_length,
key_length,
expected_inner_precise,
):
expected_out = torch.randn(1, query_length, 2, 4)
fusion_attention = Mock(return_value=(expected_out, None, None, None, 0, 0, 0))
fake_torch_npu = _fake_torch_npu(monkeypatch, npu_fusion_attention=fusion_attention)
impl = FlashAttentionImpl(num_heads=8, head_size=4, softmax_scale=0.5, causal=True)
query = torch.randn(1, query_length, 2, 4)
key = torch.randn(1, key_length, 2, 4)
out = impl.forward_fa_npu(query, key, key)
fake_torch_npu.npu_fusion_attention.assert_called_once()
args, kwargs = fake_torch_npu.npu_fusion_attention.call_args
assert torch.equal(args[0], query)
assert torch.equal(args[1], key)
assert torch.equal(args[2], key)
assert all(tensor.is_contiguous() for tensor in args[:3])
assert kwargs["head_num"] == query.shape[2]
assert kwargs["input_layout"] == "BSND"
assert kwargs["scale"] == 0.5
assert kwargs["keep_prob"] == 1.0
assert kwargs["sparse_mode"] == 3
assert kwargs["inner_precise"] == expected_inner_precise
block_mask = kwargs["atten_mask"]
assert block_mask.shape == (2048, 2048)
assert block_mask.dtype == torch.bool
assert block_mask.is_contiguous()
expected_corner = torch.tensor(
[
[False, True, True, True],
[False, False, True, True],
[False, False, False, True],
[False, False, False, False],
]
)
assert torch.equal(block_mask[:4, :4], expected_corner)
assert out is expected_out
@pytest.mark.cpu
def test_npu_causal_composes_explicit_keep_mask(monkeypatch):
expected_out = torch.randn(1, 4, 2, 4)
fusion_attention = Mock(return_value=(expected_out, None, None, None, 0, 0, 0))
fake_torch_npu = _fake_torch_npu(monkeypatch, npu_fusion_attention=fusion_attention)
impl = FlashAttentionImpl(num_heads=2, head_size=4, softmax_scale=0.5, causal=True)
query = torch.randn(1, 4, 2, 4)
key_keep_mask = torch.tensor([[True, False, True, True]])
out = impl.forward_fa_npu(
query,
query,
query,
AttentionMetadata(attn_mask=key_keep_mask),
)
fake_torch_npu.npu_fusion_attention.assert_called_once()
kwargs = fake_torch_npu.npu_fusion_attention.call_args.kwargs
assert kwargs["sparse_mode"] == 1
assert kwargs["inner_precise"] == 2
expected_block_mask = torch.tensor(
[
[
[
[False, True, True, True],
[False, True, True, True],
[False, True, False, True],
[False, True, False, False],
]
]
]
)
assert torch.equal(kwargs["atten_mask"], expected_block_mask)
assert kwargs["atten_mask"].is_contiguous()
assert out is expected_out
@pytest.mark.cpu
def test_npu_noncausal_without_explicit_mask_stays_unmasked(monkeypatch):
captured: dict = {}
def fake_attention_forward(query, key, value, **kwargs):
captured.update(kwargs)
return torch.zeros_like(query)
_fake_mindiesd(monkeypatch, attention_forward=fake_attention_forward)
query = torch.randn(1, 4, 2, 4)
_npu_impl(causal=False).forward_fa_npu(query, query, query)
assert captured["attn_mask"] is None
# --- Test group D: laser input pre-scaling in _forward_prefix_kv_slice_npu --
@pytest.mark.cpu
def test_prefix_kv_slice_applies_laser_input_scaling(monkeypatch):
monkeypatch.setenv("MINDIE_SD_FA_TYPE", "ascend_laser_attention")
captured: dict = {}
@@ -726,6 +860,7 @@ def test_prefix_kv_slice_applies_laser_input_scaling(monkeypatch):
torch.testing.assert_close(out, torch.ones(1, 8, 2, 4) * 256.0)
@pytest.mark.cpu
def test_prefix_kv_slice_no_scaling_without_factor(monkeypatch):
monkeypatch.setenv("MINDIE_SD_FA_TYPE", "ascend_laser_attention")
captured: dict = {}
@@ -0,0 +1,118 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
import torch
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionImpl
from vllm_omni.platforms import current_omni_platform
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.npu]
def _bottom_right_causal_reference(
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
scale: float,
key_keep_mask: torch.Tensor | None = None,
) -> torch.Tensor:
scores = torch.einsum("bqhd,bkhd->bhqk", query.float(), key.float()) * scale
query_positions = torch.arange(query.shape[1]).unsqueeze(1)
key_positions = torch.arange(key.shape[1]).unsqueeze(0)
keep_mask = key_positions <= query_positions + (key.shape[1] - query.shape[1])
if key_keep_mask is not None:
assert query.shape[0] == 1
keep_mask = keep_mask & key_keep_mask[0].bool().unsqueeze(0)
valid_rows = keep_mask.any(dim=-1)
output = torch.zeros(
query.shape[0],
query.shape[1],
query.shape[2],
value.shape[-1],
dtype=torch.float32,
)
valid_scores = scores[:, :, valid_rows].masked_fill(~keep_mask[valid_rows], float("-inf"))
probabilities = torch.softmax(valid_scores, dim=-1)
output[:, valid_rows] = torch.einsum("bhqk,bkhd->bqhd", probabilities, value.float())
return output
@pytest.mark.skipif(not current_omni_platform.is_npu(), reason="Native causal attention requires Ascend NPU")
def test_npu_bottom_right_causal_fully_masked_rows_are_zero():
torch.manual_seed(0)
batch_size, query_length, key_length = 1, 4, 2
num_heads, head_size = 2, 64
scale = head_size**-0.5
query_cpu = torch.randn(batch_size, query_length, num_heads, head_size).to(torch.bfloat16)
key_cpu = torch.randn(batch_size, key_length, num_heads, head_size).to(torch.bfloat16)
value_cpu = torch.randn(batch_size, key_length, num_heads, head_size).to(torch.bfloat16)
expected = _bottom_right_causal_reference(query_cpu, key_cpu, value_cpu, scale)
device = torch.device(current_omni_platform.device_type)
query = query_cpu.to(device)
key = key_cpu.to(device)
value = value_cpu.to(device)
impl = FlashAttentionImpl(
num_heads=num_heads,
head_size=head_size,
softmax_scale=scale,
causal=True,
)
output = impl.forward_fa_npu(query, key, value)
output_cpu = output.float().cpu()
assert torch.isfinite(output_cpu).all()
assert torch.equal(output_cpu[:, : query_length - key_length], torch.zeros_like(output_cpu[:, :2]))
torch.testing.assert_close(
output_cpu[:, query_length - key_length :],
expected[:, query_length - key_length :],
rtol=2e-2,
atol=2e-2,
)
@pytest.mark.skipif(not current_omni_platform.is_npu(), reason="Native causal attention requires Ascend NPU")
def test_npu_bottom_right_causal_composes_explicit_keep_mask():
torch.manual_seed(1)
batch_size, sequence_length = 1, 4
num_heads, head_size = 2, 64
scale = head_size**-0.5
key_keep_mask_cpu = torch.tensor([[True, False, True, True]])
query_cpu = torch.randn(batch_size, sequence_length, num_heads, head_size).to(torch.bfloat16)
key_cpu = torch.randn(batch_size, sequence_length, num_heads, head_size).to(torch.bfloat16)
value_cpu = torch.randn(batch_size, sequence_length, num_heads, head_size).to(torch.bfloat16)
expected = _bottom_right_causal_reference(
query_cpu,
key_cpu,
value_cpu,
scale,
key_keep_mask_cpu,
)
device = torch.device(current_omni_platform.device_type)
query = query_cpu.to(device)
key = key_cpu.to(device)
value = value_cpu.to(device)
impl = FlashAttentionImpl(
num_heads=num_heads,
head_size=head_size,
softmax_scale=scale,
causal=True,
)
output = impl.forward_fa_npu(
query,
key,
value,
AttentionMetadata(attn_mask=key_keep_mask_cpu.to(device)),
)
output_cpu = output.float().cpu()
assert torch.isfinite(output_cpu).all()
torch.testing.assert_close(output_cpu, expected, rtol=2e-2, atol=2e-2)
@@ -0,0 +1,136 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
from vllm_omni.diffusion.attention import layer
from vllm_omni.diffusion.attention.backends.fastvideo_vsa import FastVideoVSABackend, FastVideoVSAImpl
from vllm_omni.diffusion.attention.backends.sdpa import SDPABackend
from vllm_omni.diffusion.attention.parallel.base import NoParallelAttention
from vllm_omni.diffusion.data import AttentionSpec
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
class _SpecializedImpl(FastVideoVSAImpl):
pass
@pytest.fixture
def local_attention(monkeypatch):
monkeypatch.setattr(layer, "get_current_diffusion_config_or_none", lambda: None)
monkeypatch.setattr(layer, "build_parallel_attention_strategy", lambda **kwargs: NoParallelAttention())
@pytest.mark.parametrize("explicit", [False, True])
@pytest.mark.parametrize("override", [False, True])
def test_specialization_preserves_selected_backend_and_options(mocker, local_attention, explicit, override):
spec = AttentionSpec(backend="FASTVIDEO_VSA", fastvideo_vsa_topk=7) if explicit else None
select = mocker.patch.object(layer, "get_attn_backend_for_role", return_value=(FastVideoVSABackend, spec))
attention = layer.Attention(
num_heads=2,
head_size=8,
softmax_scale=8**-0.5,
causal=False,
role="video.self",
role_category="self",
qkv_layout="BSND",
impl_overrides={"FASTVIDEO_VSA": _SpecializedImpl} if override else None,
)
assert attention.attn_backend is FastVideoVSABackend
assert type(attention.attention) is (_SpecializedImpl if override else FastVideoVSAImpl)
assert attention.attention.topk == (7 if explicit else 64)
assert attention.attention.qkv_layout == "BSND"
assert attention.backend_pref == "FASTVIDEO_VSA"
assert attention.backend_explicit is explicit
assert attention.attn_spec is spec
assert isinstance(attention.parallel_strategy, NoParallelAttention)
select.assert_called_once_with(
role="video.self", head_size=8, attention_config=None, role_category="self", allow_trtllm_default=False
)
def test_unselected_specialization_does_not_replace_dense_backend(mocker, local_attention):
mocker.patch.object(layer, "get_attn_backend_for_role", return_value=(SDPABackend, None))
attention = layer.Attention(
num_heads=2,
head_size=8,
softmax_scale=8**-0.5,
causal=False,
impl_overrides={"FASTVIDEO_VSA": _SpecializedImpl},
)
assert attention.attn_backend is SDPABackend
assert type(attention.attention) is SDPABackend.get_impl_cls()
def test_specialization_does_not_bypass_selection_errors(mocker, local_attention):
mocker.patch.object(layer, "get_attn_backend_for_role", side_effect=ImportError("optional kernel unavailable"))
with pytest.raises(ImportError, match="optional kernel unavailable"):
layer.Attention(
num_heads=2,
head_size=8,
softmax_scale=8**-0.5,
causal=False,
impl_overrides={"FASTVIDEO_VSA": _SpecializedImpl},
)
def test_specialization_must_extend_selected_implementation(mocker, local_attention):
mocker.patch.object(layer, "get_attn_backend_for_role", return_value=(FastVideoVSABackend, None))
with pytest.raises(TypeError, match="must subclass the selected implementation"):
layer.Attention(
num_heads=2,
head_size=8,
softmax_scale=8**-0.5,
causal=False,
impl_overrides={"FASTVIDEO_VSA": SDPABackend.get_impl_cls()},
)
def test_platform_backend_capabilities_survive_model_specialization(mocker, local_attention):
from vllm_omni.diffusion.models.minimax_h3.attention.fastvideo_h3 import MiniMaxH3VSAImpl
class PlatformBackend(FastVideoVSABackend):
@classmethod
def supports_packed_mask_free(cls):
return False
@staticmethod
def get_supported_head_sizes():
return [128]
# A registry/platform may refine capabilities without changing the
# implementation. The model must not replace those capability decisions.
mocker.patch.object(layer, "get_attn_backend_for_role", return_value=(PlatformBackend, None))
attention = layer.Attention(
num_heads=2,
head_size=128,
softmax_scale=128**-0.5,
causal=False,
impl_overrides={"FASTVIDEO_VSA": MiniMaxH3VSAImpl},
)
assert attention.attn_backend is PlatformBackend
assert not attention.attn_backend.supports_packed_mask_free()
assert attention.attn_backend.get_supported_head_sizes() == [128]
assert type(attention.attention) is MiniMaxH3VSAImpl
def test_incompatible_platform_implementation_is_not_discarded(mocker, local_attention):
class PlatformImpl(FastVideoVSAImpl):
pass
class PlatformBackend(FastVideoVSABackend):
@staticmethod
def get_impl_cls():
return PlatformImpl
mocker.patch.object(layer, "get_attn_backend_for_role", return_value=(PlatformBackend, None))
with pytest.raises(TypeError, match="selected implementation .*PlatformImpl"):
layer.Attention(
num_heads=2,
head_size=8,
softmax_scale=8**-0.5,
causal=False,
impl_overrides={"FASTVIDEO_VSA": _SpecializedImpl},
)
@@ -0,0 +1,160 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Cross-repository CPU dispatch contracts, requiring the matching MindIE-SD source.
Only native operators are replaced. Public adapters, rotations, layouts, padding,
scale preparation, sequence metadata and output cropping execute their real code.
Operator substitutes carry floating tensors; these are not numerical quantization tests.
"""
import importlib
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from vllm_omni.diffusion.attention.backends import flash_attn, rainfusion_attn
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.platforms.npu.quant.kv_quant_npu import get_quant_attention_rotation
mindiesd = pytest.importorskip("mindiesd")
torch_npu = importlib.import_module("torch_npu")
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
@pytest.mark.parametrize(
"precision,batch,q_len,kv_len,q_heads",
[
("fp8", 1, 130, 130, 2),
("mxfp8", 1, 130, 130, 2),
("mxfp4", 1, 130, 130, 2),
("mxfp8", 2, 17, 8, 4),
("mxfp8", 2, 8, 17, 4),
],
)
@pytest.mark.parametrize("layout", ["BSND", "BNSD"])
def test_omni_calls_real_dense_public_api(monkeypatch, precision, layout, batch, q_len, kv_len, q_heads):
assert callable(getattr(mindiesd, "quant_attention", None)), "MindIE-SD must export quant_attention"
monkeypatch.setattr(flash_attn, "current_omni_platform", SimpleNamespace(is_npu=lambda: True, device_name="npu"))
monkeypatch.setattr(flash_attn, "get_current_diffusion_config_or_none", lambda: None)
quantized_inputs = []
def dynamic_quant(tensor, **kwargs):
quantized_inputs.append((tensor.clone(), kwargs))
shape = list(tensor.shape)
axis = kwargs.get("axis", -1)
shape[axis] = max(1, shape[axis] // 32)
return tensor, torch.ones(shape, dtype=torch.uint8)
monkeypatch.setattr(torch_npu, "npu_dynamic_block_quant", dynamic_quant, raising=False)
monkeypatch.setattr(torch_npu, "npu_dynamic_mx_quant", dynamic_quant, raising=False)
execute = Mock(side_effect=lambda query, *args, **kwargs: (query, None))
metadata = Mock(return_value=torch.zeros(1, dtype=torch.int32))
if precision == "fp8":
monkeypatch.setattr(torch.ops.mindiesd, "fused_infer_attention_score_v2", execute, raising=False)
elif precision == "mxfp8":
monkeypatch.setattr(torch_npu, "npu_fused_infer_attention_score_v2", execute, raising=False)
else:
monkeypatch.setattr(torch.ops.mindiesd, "quant_flash_attn", execute, raising=False)
monkeypatch.setattr(torch.ops.mindiesd, "quant_flash_attn_metadata", metadata, raising=False)
query = torch.randn(batch, q_len, q_heads, 64, dtype=torch.bfloat16)
key = torch.randn(batch, kv_len, 2, 64, dtype=torch.bfloat16)
value = torch.randn_like(key)
if layout == "BNSD":
query, key, value = (tensor.transpose(1, 2) for tensor in (query, key, value))
impl = flash_attn.FlashAttentionImpl(
num_heads=q_heads,
head_size=64,
softmax_scale=0.37,
qkv_layout=layout,
)
output = impl.forward_npu(query, key, value, AttentionMetadata(extra={"kv_cache_dtype": precision}))
rotation = get_quant_attention_rotation(query.device, query.dtype, 64)
expected = query @ rotation
torch.testing.assert_close(output, expected)
assert output.shape == query.shape and len(quantized_inputs) == 3
execute.assert_called_once()
kwargs = execute.call_args.kwargs
assert kwargs["softmax_scale"] == 0.37
if precision == "fp8":
assert kwargs["input_layout"] == "BNSD"
assert quantized_inputs[0][0].shape == (2, 130, 64)
assert quantized_inputs[0][1]["row_block_size"] == 128
elif precision == "mxfp8":
args = execute.call_args.args
assert args[0].shape == (batch * q_len, q_heads, 64)
assert args[1].shape == args[2].shape == (batch * kv_len, 2, 64)
assert kwargs["input_layout"] == "TND"
assert kwargs["num_query_heads"] == q_heads and kwargs["num_key_value_heads"] == 2
assert kwargs["actual_seq_qlen"] == [q_len * (i + 1) for i in range(batch)]
assert kwargs["actual_seq_kvlen"] == [kv_len * (i + 1) for i in range(batch)]
assert [entry[1]["axis"] for entry in quantized_inputs] == [-1, -1, 0]
else:
seq_axis = 1 if layout == "BSND" else 2
assert quantized_inputs[0][0].shape[seq_axis] == 512
assert kwargs["seqused_q"].tolist() == kwargs["seqused_kv"].tolist() == [512]
assert kwargs["layout_q"] == kwargs["layout_out"] == layout
assert kwargs["q_dtype"] == torch_npu.float4_e2m1fn_x2
assert kwargs["q_descale_dtype"] == torch_npu.float8_e8m0fnu
assert execute.call_args.args[3].shape[:2] == (1, 2)
assert [entry[1]["axis"] for entry in quantized_inputs] == [-1, -1, seq_axis]
assert kwargs["metadata"] is metadata.return_value
metadata.assert_called_once()
@pytest.mark.parametrize("precision", ["fp8", "mxfp4"])
@pytest.mark.parametrize("length", [65, 600])
def test_omni_calls_real_sparse_public_api(monkeypatch, precision, length):
rf = importlib.import_module("mindiesd.layers.flash_attn.sparse_flash_attn_rf_v3")
monkeypatch.setattr(flash_attn, "current_omni_platform", SimpleNamespace(is_npu=lambda: True, device_name="npu"))
for module in (flash_attn, rainfusion_attn):
monkeypatch.setattr(module, "get_current_diffusion_config_or_none", lambda: None)
rainfusion_attn._mindiesd_supports_precision.cache_clear()
monkeypatch.setattr(rf, "do_tensor_rearrange_only", lambda q, k, v, *a, **kw: (q, k, v))
monkeypatch.setattr(rf, "avgpool", lambda *a, **kw: torch.ones(1))
monkeypatch.setattr(rf, "_generate_mask_direct", lambda *a, **kw: torch.ones(1, 2, 8, 8))
monkeypatch.setattr(rf, "_bsa_inv_rearrange", lambda out, *a: out)
def quantize(tensor, **kwargs):
shape = list(tensor.shape)
axis = kwargs.get("axis", -1)
shape[axis] = max(1, shape[axis] // 32)
return tensor, torch.ones(shape, dtype=torch.uint8)
monkeypatch.setattr(torch_npu, "npu_dynamic_mx_quant", quantize, raising=False)
monkeypatch.setattr(torch_npu, "npu_dynamic_block_quant", quantize, raising=False)
execute = Mock(side_effect=lambda **kw: (torch.ones_like(kw["query"]), None))
monkeypatch.setattr(torch.ops.mindiesd, "block_sparse_attention", execute, raising=False)
query = torch.randn(1, length, 2, 64, dtype=torch.bfloat16)
impl = rainfusion_attn.RainFusionAttentionImpl(
num_heads=2,
head_size=64,
softmax_scale=0.37,
qkv_layout="BSND",
backend_kwargs={"sparsity": 0.8},
)
plan = rainfusion_attn.RainFusionPlan(used_len=length, prefix_len=0, latent_shape=(1, 1, length))
try:
output = impl._forward_sparse_npu(
query,
query,
query,
plan,
AttentionMetadata(extra={"kv_cache_dtype": precision}),
)
finally:
rainfusion_attn._mindiesd_supports_precision.cache_clear()
torch.testing.assert_close(output, torch.ones_like(query))
execute.assert_called_once()
kwargs = execute.call_args.kwargs
assert kwargs["actual_seq_lengths"] == kwargs["actual_seq_lengths_kv"] == [length]
assert kwargs["q_input_layout"] == "BNSD"
assert kwargs["block_sparse_mask"] is not None
if precision == "mxfp4":
assert kwargs["query"].shape[2] == (length + 63) // 64 * 64
assert kwargs["quant_mode"] == 2
assert kwargs["q_dtype"] == torch_npu.float4_e2m1fn_x2
assert kwargs["v_scale_dtype"] == torch_npu.float8_e8m0fnu
@@ -0,0 +1,323 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CPU contract tests for MindIE adapters; no NPU kernels are mocked in production."""
import sys
from types import ModuleType, SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from vllm_omni.diffusion.attention import layer as layer_mod
from vllm_omni.diffusion.attention.backends import flash_attn, rainfusion_attn
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata, VideoTokenLayout
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
_DEFAULT_SKIP_STEPS = frozenset({0, 1, 38, 39})
@pytest.fixture
def runtime(monkeypatch):
mod = ModuleType("mindiesd")
def execute(q, k, v, *, precision, layout, scale, q_rot=None, k_rot=None):
return q
setattr(mod, "quant_attention", Mock(side_effect=execute))
monkeypatch.setitem(sys.modules, mod.__name__, mod)
for module in (flash_attn, layer_mod):
monkeypatch.setattr(module, "current_omni_platform", SimpleNamespace(is_npu=lambda: True, device_name="npu"))
monkeypatch.setattr(flash_attn, "get_current_diffusion_config_or_none", lambda: None)
monkeypatch.setattr(rainfusion_attn, "get_current_diffusion_config_or_none", lambda: None)
monkeypatch.setattr(rainfusion_attn, "is_forward_context_available", lambda: False)
supports_precision = rainfusion_attn._mindiesd_supports_precision
supports_precision.cache_clear()
yield mod
supports_precision.cache_clear()
def flash(layout="BSND"):
return flash_attn.FlashAttentionImpl(
num_heads=2,
head_size=64,
softmax_scale=0.37,
qkv_layout=layout,
)
def sparse(**kwargs):
return rainfusion_attn.RainFusionAttentionImpl(
num_heads=2,
head_size=64,
softmax_scale=0.37,
qkv_layout="BSND",
backend_kwargs={"sparsity": 0.8, **kwargs},
)
def video_metadata(**extra):
return AttentionMetadata(
video_layout=VideoTokenLayout(prefix_len=0, latent_grid=(4, 32, 32), used_len=4096),
extra={"max_seqlen_q": 4096, **extra},
)
@pytest.mark.parametrize("method", ["fp8", "mxfp8", "mxfp4"])
@pytest.mark.parametrize("layout", [None, "BSND", "BNSD"])
def test_exact_runtime_layout_scale_and_rotation(runtime, method, layout):
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
if layout == "BNSD":
q = q.transpose(1, 2)
impl = flash(layout=layout)
metadata = AttentionMetadata(extra={"kv_cache_dtype": method})
assert impl.forward_npu(q, q, q, metadata) is q
args, kwargs = runtime.quant_attention.call_args
assert args[0] is q # no unconditional transpose or extra quantization
assert kwargs["layout"] == (layout or "BSND") and kwargs["scale"] == 0.37 and kwargs["precision"] == method
assert "attn_mask" not in kwargs
from vllm_omni.platforms.npu.quant.kv_quant_npu import get_quant_attention_rotation
assert kwargs["q_rot"] is kwargs["k_rot"]
torch.testing.assert_close(kwargs["q_rot"], get_quant_attention_rotation(q.device, q.dtype, 64))
def test_missing_runtime_requires_config_change(runtime):
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
del runtime.quant_attention
with pytest.raises(ImportError, match="diffusion_kv_cache_dtype"):
flash().forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": "mxfp4"}))
def test_operator_failure_never_falls_back(runtime):
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
runtime.quant_attention.side_effect = RuntimeError("operator failure")
with pytest.raises(RuntimeError, match="operator failure"):
flash().forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": "mxfp4"}))
runtime.quant_attention.assert_called_once()
@pytest.mark.parametrize("method", ["fp8", "mxfp8", "mxfp4"])
def test_unsupported_shape_requires_config_change(runtime, method):
q = torch.randn(1, 128, 2, 96, dtype=torch.bfloat16)
with pytest.raises(ValueError, match="diffusion_kv_cache_dtype"):
flash().forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": method}))
runtime.quant_attention.assert_not_called()
def test_packed_metadata_never_reaches_quant_runtime(runtime):
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
metadata = AttentionMetadata(extra={"kv_cache_dtype": "mxfp8", "cu_seqlens_q": torch.tensor([0, 64, 128])})
with pytest.raises(ValueError, match="packed/varlen"):
flash().forward_npu(q, q, q, metadata)
runtime.quant_attention.assert_not_called()
def make_layer(runtime, monkeypatch, method="mxfp8", skip_steps=_DEFAULT_SKIP_STEPS):
layer = object.__new__(layer_mod.Attention)
torch.nn.Module.__init__(layer)
layer.attention = flash()
layer.attn_backend = flash_attn.FlashAttentionBackend
layer._disable_kv_quant = False
layer.layer_idx = 3
cfg = SimpleNamespace(
diffusion_kv_cache_dtype=method,
diffusion_kv_cache_skip_step_indices=skip_steps,
diffusion_kv_cache_skip_layer_indices=None,
parallel_config=SimpleNamespace(ring_degree=1),
)
layer._init_kv_cache_quantization(cfg)
return layer
@pytest.mark.parametrize("method", ["fp8", "mxfp8", "mxfp4"])
def test_global_quant_method_is_published_per_forward(runtime, monkeypatch, method):
layer = make_layer(runtime, monkeypatch, method=method, skip_steps=None)
assert layer._with_kv_cache_dtype(None).extra["kv_cache_dtype"] == method
@pytest.mark.parametrize("method", [None, "auto", "float"])
def test_optout_without_quant_policy_keeps_metadata_unchanged(runtime, monkeypatch, method):
layer = make_layer(runtime, monkeypatch)
layer._disable_kv_quant = True
layer._init_kv_cache_quantization(
SimpleNamespace(
diffusion_kv_cache_dtype=method,
diffusion_kv_cache_skip_step_indices=None,
diffusion_kv_cache_skip_layer_indices=None,
parallel_config=SimpleNamespace(ring_degree=1),
)
)
source = video_metadata()
assert layer._with_kv_cache_dtype(source) is source
def test_step_skip_is_request_local_and_cross_optout(runtime, monkeypatch):
layer = make_layer(runtime, monkeypatch)
ctx = SimpleNamespace(denoise_step_idx=0)
monkeypatch.setattr(layer_mod, "is_forward_context_available", lambda: True)
monkeypatch.setattr(layer_mod, "get_forward_context", lambda: ctx)
source = video_metadata()
for step in list(range(40)) * 2:
ctx.denoise_step_idx = step
resolved = layer._with_kv_cache_dtype(source)
assert resolved.extra["kv_cache_dtype"] == ("float" if step in (0, 1, 38, 39) else "mxfp8")
assert resolved.video_layout is source.video_layout
assert source.extra == {"max_seqlen_q": 4096}
layer._disable_kv_quant = True
assert layer._with_kv_cache_dtype(source).extra["kv_cache_dtype"] == "float"
def test_ring_attention_is_rejected(runtime, monkeypatch):
layer = make_layer(runtime, monkeypatch)
cfg = SimpleNamespace(diffusion_kv_cache_dtype="mxfp8", parallel_config=SimpleNamespace(ring_degree=2))
with pytest.raises(ValueError, match="ring"):
layer._init_kv_cache_quantization(cfg)
def test_sparse_tail_steps_and_dense_quant_dispatch(runtime, monkeypatch):
ctx = SimpleNamespace(denoise_step_idx=49, total_denoise_steps=50)
monkeypatch.setattr(rainfusion_attn, "is_forward_context_available", lambda: True)
monkeypatch.setattr(rainfusion_attn, "get_forward_context", lambda: ctx)
impl = sparse(start_step=2, end_step=2)
metadata = video_metadata(kv_cache_dtype="fp8")
assert impl._resolve_plan(metadata) is None
q = torch.randn(1, 4096, 2, 64, dtype=torch.bfloat16)
impl.forward_npu(q, q, q, metadata)
assert runtime.quant_attention.call_args.kwargs["precision"] == "fp8"
@pytest.mark.parametrize("method", ["fp8", "mxfp4"])
def test_sparse_quant_uses_public_runtime(runtime, monkeypatch, method):
runtime.sparse_attention = Mock(side_effect=lambda q, k, v, **kwargs: q)
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: True)
q = torch.randn(1, 4224, 2, 64, dtype=torch.bfloat16)
out = sparse().forward_npu(q, q, q, video_metadata(kv_cache_dtype=method))
# The sparse path reuses the query tail as the output padding instead of
# zero-filling a fresh buffer; rows past the 4096-row valid prefix are
# never read back downstream (the model unpads by cu_seqlens), so pin the
# passthrough rather than requiring zeros.
assert out.shape == q.shape and torch.equal(out[:, 4096:], q[:, 4096:])
kwargs = runtime.sparse_attention.call_args.kwargs
assert (kwargs["precision"], kwargs["sparse_type"], kwargs["inner_precise"]) == (method, "rf_v3", 4)
runtime.quant_attention.assert_not_called()
def test_sparse_custom_mask_stays_dense(runtime):
metadata = AttentionMetadata(
attn_mask=torch.ones(1, 4096, dtype=torch.bool),
video_layout=VideoTokenLayout(prefix_len=0, latent_grid=(4, 32, 32)),
extra={"max_seqlen_q": 4096},
)
assert sparse()._resolve_plan(metadata) is None
def test_sparse_model_padding_mask_is_equivalent_to_trimming(runtime):
metadata = video_metadata()
metadata.attn_mask = torch.arange(4224)[None] < 4096
assert sparse()._resolve_plan(metadata).used_len == 4096
def test_bsa_operator_failure_never_retries(runtime, monkeypatch):
runtime.sparse_attention = Mock(side_effect=RuntimeError("BSA operator failure"))
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: True)
q = torch.randn(1, 4096, 2, 64, dtype=torch.bfloat16)
with pytest.raises(RuntimeError, match="BSA operator failure"):
sparse().forward_npu(q, q, q, video_metadata(kv_cache_dtype="mxfp4"))
runtime.sparse_attention.assert_called_once()
@pytest.mark.parametrize(
"method,shape,supports_precision,error",
[
("mxfp8", (1, 4096, 2, 64), True, "sparse mxfp8 is not supported"),
("mxfp4", (1, 4096, 2, 64), False, "explicitly support precision"),
("fp8", (2, 4096, 2, 64), True, "batch size 1"),
("fp8", (1, 4096, 2, 32), True, "power-of-two head dimension"),
("mxfp4", (1, 4096, 2, 96), True, "power-of-two head dimension"),
],
)
def test_invalid_sparse_quant_requires_config_change(runtime, monkeypatch, method, shape, supports_precision, error):
runtime.sparse_attention = Mock(side_effect=lambda q, k, v, **kwargs: q)
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: supports_precision)
q = torch.randn(shape, dtype=torch.bfloat16)
with pytest.raises(ValueError, match=error):
sparse().forward_npu(q, q, q, video_metadata(kv_cache_dtype=method))
runtime.sparse_attention.assert_not_called()
def test_layer_selector_requires_index_except_cross_optout(runtime, monkeypatch):
layer = make_layer(runtime, monkeypatch)
layer.layer_idx = None
config = SimpleNamespace(
diffusion_kv_cache_dtype="mxfp8",
diffusion_kv_cache_skip_step_indices=None,
diffusion_kv_cache_skip_layer_indices={0, 3},
parallel_config=SimpleNamespace(ring_degree=1),
)
with pytest.raises(ValueError, match="parseable transformer block index"):
layer._init_kv_cache_quantization(config)
layer._disable_kv_quant = True
layer._init_kv_cache_quantization(config)
assert layer._with_kv_cache_dtype(None).extra["kv_cache_dtype"] == "float"
@pytest.mark.parametrize("prefix", ["transformer.blocks.3.attn1", "transformer_2.blocks.3.attn1"])
@pytest.mark.parametrize("method", ["fp8", "mxfp4"])
def test_expert_layer_and_step_skip_remain_sparse(runtime, monkeypatch, prefix, method):
def execute(q, k, v, *, precision="bf16", **kwargs):
return q
runtime.sparse_attention = Mock(wraps=execute)
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: True)
layer = make_layer(runtime, monkeypatch)
layer.layer_idx = rainfusion_attn._try_extract_layer_index(prefix)
assert layer.layer_idx == 3
config = SimpleNamespace(
diffusion_kv_cache_dtype=method,
diffusion_kv_cache_skip_step_indices={0, 1, 38, 39},
diffusion_kv_cache_skip_layer_indices={3},
parallel_config=SimpleNamespace(ring_degree=1),
)
impl = sparse()
layer.attention = impl
layer.attn_backend = rainfusion_attn.RainFusionAttentionBackend
layer._init_kv_cache_quantization(config)
ctx = SimpleNamespace(denoise_step_idx=0)
monkeypatch.setattr(layer_mod, "is_forward_context_available", lambda: True)
monkeypatch.setattr(layer_mod, "get_forward_context", lambda: ctx)
source = video_metadata(kv_cache_dtype="mxfp8")
q = torch.randn(1, 4096, 2, 64, dtype=torch.bfloat16)
for layer_idx, step in ((3, 0), (3, 2), (4, 0), (4, 1), (4, 2), (4, 37), (4, 38), (4, 39), (4, 0)):
expected = "bf16" if layer_idx == 3 or step in (0, 1, 38, 39) else method
layer.layer_idx = layer_idx
ctx.denoise_step_idx = step
metadata = layer._with_kv_cache_dtype(source)
impl.forward_npu(q, q, q, metadata)
assert runtime.sparse_attention.call_args.kwargs["precision"] == expected
runtime.quant_attention.assert_not_called()
assert source.extra["kv_cache_dtype"] == "mxfp8"
def test_custom_attention_initialization_keeps_upstream_optout(monkeypatch):
monkeypatch.setattr(layer_mod, "get_current_diffusion_config_or_none", lambda: None)
monkeypatch.setattr(layer_mod, "build_parallel_attention_strategy", lambda **kw: object())
monkeypatch.setattr(layer_mod, "NoParallelAttention", lambda: object())
custom = torch.nn.Identity()
layer = layer_mod.Attention(
num_heads=2,
head_size=64,
softmax_scale=0.125,
causal=False,
custom_attention=custom,
skip_sequence_parallel=True,
)
assert layer.attention is custom and layer._kv_cache_dtype is None
def test_legacy_global_fp8_uses_implicit_bsnd(runtime):
q = torch.randn(1, 17, 2, 64, dtype=torch.bfloat16)
impl = flash(layout=None)
assert impl.forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": "fp8"})) is q
assert runtime.quant_attention.call_args.kwargs["layout"] == "BSND"
assert runtime.quant_attention.call_args.kwargs["precision"] == "fp8"
@@ -133,18 +133,17 @@ def test_invalid_ref2va_video_spans_fall_back_to_dense():
assert make_impl()._resolve_plan(AttentionMetadata(extra={"max_seqlen_q": 12000}, video_layout=layout)) is None
def test_validate_available_rejects_legacy_mindiesd(monkeypatch):
def test_validate_available_accepts_single_video_mindiesd(monkeypatch):
mindiesd = types.ModuleType("mindiesd")
def sparse_attention(query, key, value, **kwargs):
return query
mindiesd.sparse_attention = sparse_attention
monkeypatch.setattr(mindiesd, "sparse_attention", sparse_attention, raising=False)
monkeypatch.setitem(sys.modules, "mindiesd", mindiesd)
monkeypatch.setattr("importlib.util.find_spec", lambda _: object())
with pytest.raises(ValueError, match="video_spans"):
RainFusionAttentionBackend.validate_available()
RainFusionAttentionBackend.validate_available()
def test_validate_available_accepts_new_mindiesd(monkeypatch):
@@ -153,13 +152,25 @@ def test_validate_available_accepts_new_mindiesd(monkeypatch):
def sparse_attention(query, key, value, *, video_spans=None, **kwargs):
return query
mindiesd.sparse_attention = sparse_attention
monkeypatch.setattr(mindiesd, "sparse_attention", sparse_attention, raising=False)
monkeypatch.setitem(sys.modules, "mindiesd", mindiesd)
monkeypatch.setattr("importlib.util.find_spec", lambda _: object())
RainFusionAttentionBackend.validate_available()
def test_prefix_kv_slicing_contract_is_pinned():
# Regression pin for the #5543 packed-padding crash: reverting the flag
# would pass CI while reintroducing the ValueError for non-64-aligned
# requests. Both halves of the contract must hold together — the impl
# trims [real, pad] packed tensors to the valid prefix itself (so it may
# advertise prefix slicing) and it never reads attn_mask (so it must keep
# refusing one), which is what lets the model skip materializing the
# padding mask that _assert_metadata_compatible would reject.
assert RainFusionAttentionBackend.supports_prefix_kv_slicing is True
assert RainFusionAttentionBackend.supports_attention_mask() is False
@pytest.mark.parametrize("grid", [(4, 24, 40), (1, 24, 40)])
def test_short_video_stays_dense(grid):
assert make_impl()._resolve_plan(make_metadata(grid)) is None
@@ -300,22 +311,21 @@ def _fake_mindiesd_module():
import types
fake = types.ModuleType("mindiesd")
fake.sparse_attention = lambda *args, **kwargs: None
setattr(fake, "sparse_attention", lambda *args, **kwargs: None)
return fake
def test_precision_non_bf16_requires_mindiesd_support():
"""precision != bf16 against a mindiesd lacking the kwarg must raise RuntimeError."""
"""Unsupported sparse precision must fail before any operator executes."""
import sys
impl = make_impl(precision="mix")
sys.modules["mindiesd"] = _fake_mindiesd_module()
try:
with mock.patch.dict(sys.modules, {"mindiesd": _fake_mindiesd_module()}):
with mock.patch.object(rainfusion_attn, "_mindiesd_supports_precision", return_value=False):
with pytest.raises(RuntimeError, match="requires MindIE-SD"):
impl._forward_sparse_npu(None, None, None, None)
finally:
sys.modules.pop("mindiesd", None)
with pytest.raises(ValueError, match="explicitly support precision"):
impl._forward_sparse_npu(
None, None, None, RainFusionPlan(prefix_len=0, used_len=8, latent_shape=[2, 2, 2])
)
def test_precision_non_bf16_passes_gate_when_supported():
@@ -323,8 +333,7 @@ def test_precision_non_bf16_passes_gate_when_supported():
import sys
impl = make_impl(precision="mix")
sys.modules["mindiesd"] = _fake_mindiesd_module()
try:
with mock.patch.dict(sys.modules, {"mindiesd": _fake_mindiesd_module()}):
with mock.patch.object(rainfusion_attn, "_mindiesd_supports_precision", return_value=True):
# q/k/v shapes: [B, S, N, D]; plan geometry must match S.
q = torch.randn(1, 8, 4, 128)
@@ -332,5 +341,3 @@ def test_precision_non_bf16_passes_gate_when_supported():
# The gate must pass; the fake mindiesd returns None so no crash.
out = impl._forward_sparse_npu(q, q, q, plan)
assert out is None
finally:
sys.modules.pop("mindiesd", None)
@@ -0,0 +1,221 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from dataclasses import dataclass
from typing import Any
import pytest
import torch
from vllm_omni.diffusion.attention.backends import ring_flash_attn, ring_pytorch_attn
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.backends.ring.ring_selector import AttnType
from vllm_omni.diffusion.attention.backends.ring.ring_utils import ring_kv_block_valid_length
from vllm_omni.diffusion.attention.parallel import ring as ring_parallel
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
class _SingleRankComm:
rank = 0
world_size = 1
@dataclass(frozen=True)
class _FakeSequenceParallelGroup:
ring_group: object
def _fake_attention_result(q, _k, _v, **_kwargs):
return torch.zeros_like(q), torch.zeros(q.shape[0], q.shape[2], q.shape[1])
@pytest.mark.parametrize(("block_rank", "expected"), [(0, 4), (1, 1), (2, 0)])
def test_global_prefix_is_partitioned_across_ring_blocks(block_rank, expected):
assert ring_kv_block_valid_length(5, 4, block_rank, 3) == expected
def test_absent_prefix_length_keeps_the_full_ring_block():
assert ring_kv_block_valid_length(None, 4, 2, 3) == 4
def test_pytorch_ring_trims_padding_from_each_kv_block(monkeypatch):
captured: dict[str, Any] = {}
def fake_attention(q, k, v, **kwargs):
captured.update(k=k, v=v, **kwargs)
return _fake_attention_result(q, k, v, **kwargs)
monkeypatch.setattr(ring_pytorch_attn, "RingComm", lambda _group: _SingleRankComm())
monkeypatch.setattr(ring_pytorch_attn, "pytorch_attn_forward", fake_attention)
qkv = torch.randn(1, 4, 2, 8)
output = ring_pytorch_attn.ring_pytorch_attn_func(
qkv,
qkv,
qkv,
group=object(),
valid_kv_length=3,
)
assert output.shape == qkv.shape
assert captured["k"].shape[1] == 3
assert captured["v"].shape[1] == 3
def test_pytorch_ring_trims_circulated_blocks_by_source_rank(monkeypatch):
observed_lengths = []
# Each ring step sends K and V separately.
blocks = [torch.randn(1, 4, 2, 8) for _ in range(4)]
class FakeRingComm:
rank = 2
world_size = 3
def __init__(self, _group):
self._next_block = 0
def send_recv(self, _tensor):
block = blocks[self._next_block]
self._next_block += 1
return block
def commit(self):
pass
def wait(self):
pass
def fake_attention(q, k, v, **kwargs):
observed_lengths.append(k.shape[1])
return _fake_attention_result(q, k, v, **kwargs)
monkeypatch.setattr(ring_pytorch_attn, "RingComm", FakeRingComm)
monkeypatch.setattr(ring_pytorch_attn, "pytorch_attn_forward", fake_attention)
qkv = torch.randn(1, 4, 2, 8)
ring_pytorch_attn.ring_pytorch_attn_func(
qkv,
qkv,
qkv,
group=object(),
valid_kv_length=5,
)
# Rank 2 starts with global block 2, then receives blocks 1 and 0.
# The first block is empty and therefore does not launch attention.
assert observed_lengths == [1, 4]
def test_flash_ring_trims_padding_from_each_kv_block(monkeypatch):
captured: dict[str, Any] = {}
def fake_attention(q, k, v, **kwargs):
captured.update(k=k, v=v, **kwargs)
return _fake_attention_result(q, k, v, **kwargs)
monkeypatch.setattr(ring_flash_attn, "RingComm", lambda _group: _SingleRankComm())
monkeypatch.setattr(ring_flash_attn, "select_flash_attn_impl", lambda *_args, **_kwargs: fake_attention)
qkv = torch.randn(1, 4, 2, 8)
output, _ = ring_flash_attn.ring_flash_attn_forward(
object(),
qkv,
qkv,
qkv,
softmax_scale=0.25,
causal=False,
attn_type=AttnType.FA,
valid_kv_length=3,
)
assert output.shape == qkv.shape
assert captured["k"].shape[1] == 3
assert captured["v"].shape[1] == 3
def test_ring_dispatch_forwards_global_valid_kv_length(monkeypatch):
captured: dict[str, Any] = {}
def fake_ring_attention(*_args, **kwargs):
captured.update(kwargs)
return "output"
monkeypatch.setattr(ring_pytorch_attn, "ring_pytorch_attn_func", fake_ring_attention)
strategy = ring_parallel.RingParallelAttention(
_FakeSequenceParallelGroup(ring_group=object()),
attn_backend_pref="sdpa",
)
metadata = AttentionMetadata(extra={"valid_kv_length": 3})
query = torch.randn(1, 4, 2, 8)
assert strategy.run_attention(query, query, query, metadata) == "output"
assert captured["valid_kv_length"] == 3
def test_pytorch_ring_rejects_causal_with_a_valid_prefix(monkeypatch):
"""Trimming a K/V block under causal=True would silently shift the diagonal.
The query block keeps the full padded length while the circulated K/V block
is trimmed, and a causal mask over a rectangular tile is bottom-right
aligned, so the combination must be refused rather than computed.
"""
monkeypatch.setattr(ring_pytorch_attn, "RingComm", lambda _group: _SingleRankComm())
monkeypatch.setattr(ring_pytorch_attn, "pytorch_attn_forward", _fake_attention_result)
qkv = torch.randn(1, 4, 2, 8)
with pytest.raises(ValueError, match="valid_kv_length is not supported with causal=True"):
ring_pytorch_attn.ring_pytorch_attn_func(
qkv,
qkv,
qkv,
causal=True,
group=object(),
valid_kv_length=3,
)
def test_flash_ring_rejects_causal_with_a_valid_prefix(monkeypatch):
"""Same refusal on the Flash-Attention ring path."""
monkeypatch.setattr(ring_flash_attn, "RingComm", lambda _group: _SingleRankComm())
monkeypatch.setattr(
ring_flash_attn,
"select_flash_attn_impl",
lambda *_args, **_kwargs: _fake_attention_result,
)
qkv = torch.randn(1, 4, 2, 8)
with pytest.raises(ValueError, match="valid_kv_length is not supported with causal=True"):
ring_flash_attn.ring_flash_attn_forward(
object(),
qkv,
qkv,
qkv,
softmax_scale=0.25,
causal=True,
attn_type=AttnType.FA,
valid_kv_length=3,
)
def test_ring_paths_still_accept_causal_without_a_valid_prefix(monkeypatch):
"""The guard is scoped to the padded case: plain causal Ring is unchanged."""
monkeypatch.setattr(ring_flash_attn, "RingComm", lambda _group: _SingleRankComm())
monkeypatch.setattr(
ring_flash_attn,
"select_flash_attn_impl",
lambda *_args, **_kwargs: _fake_attention_result,
)
qkv = torch.randn(1, 4, 2, 8)
output, _ = ring_flash_attn.ring_flash_attn_forward(
object(),
qkv,
qkv,
qkv,
softmax_scale=0.25,
causal=True,
attn_type=AttnType.FA,
)
assert output.shape == qkv.shape