[Hardware][Ascend] Integrate MindIE dense and sparse quantized attention for Wan2.2 T2V (#7561)

Signed-off-by: zqxu <xuzhenqiang3@huawei.com>
Co-authored-by: zqxu <xuzhenqiang3@huawei.com>
This commit is contained in:
Zhenqiang Xuandzqxu authored and GitHub committed 2026-09-18 01:28:27 +00:00
1 parent f8197e120f
commit 242c3cddeb
15 files changed
+853 -368

No files matched your search

@@ -0,0 +1,160 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Cross-repository CPU dispatch contracts, requiring the matching MindIE-SD source.
Only native operators are replaced. Public adapters, rotations, layouts, padding,
scale preparation, sequence metadata and output cropping execute their real code.
Operator substitutes carry floating tensors; these are not numerical quantization tests.
"""
import importlib
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from vllm_omni.diffusion.attention.backends import flash_attn, rainfusion_attn
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.platforms.npu.quant.kv_quant_npu import get_quant_attention_rotation
mindiesd = pytest.importorskip("mindiesd")
torch_npu = importlib.import_module("torch_npu")
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
@pytest.mark.parametrize(
"precision,batch,q_len,kv_len,q_heads",
[
("fp8", 1, 130, 130, 2),
("mxfp8", 1, 130, 130, 2),
("mxfp4", 1, 130, 130, 2),
("mxfp8", 2, 17, 8, 4),
("mxfp8", 2, 8, 17, 4),
],
)
@pytest.mark.parametrize("layout", ["BSND", "BNSD"])
def test_omni_calls_real_dense_public_api(monkeypatch, precision, layout, batch, q_len, kv_len, q_heads):
assert callable(getattr(mindiesd, "quant_attention", None)), "MindIE-SD must export quant_attention"
monkeypatch.setattr(flash_attn, "current_omni_platform", SimpleNamespace(is_npu=lambda: True, device_name="npu"))
monkeypatch.setattr(flash_attn, "get_current_diffusion_config_or_none", lambda: None)
quantized_inputs = []
def dynamic_quant(tensor, **kwargs):
quantized_inputs.append((tensor.clone(), kwargs))
shape = list(tensor.shape)
axis = kwargs.get("axis", -1)
shape[axis] = max(1, shape[axis] // 32)
return tensor, torch.ones(shape, dtype=torch.uint8)
monkeypatch.setattr(torch_npu, "npu_dynamic_block_quant", dynamic_quant, raising=False)
monkeypatch.setattr(torch_npu, "npu_dynamic_mx_quant", dynamic_quant, raising=False)
execute = Mock(side_effect=lambda query, *args, **kwargs: (query, None))
metadata = Mock(return_value=torch.zeros(1, dtype=torch.int32))
if precision == "fp8":
monkeypatch.setattr(torch.ops.mindiesd, "fused_infer_attention_score_v2", execute, raising=False)
elif precision == "mxfp8":
monkeypatch.setattr(torch_npu, "npu_fused_infer_attention_score_v2", execute, raising=False)
else:
monkeypatch.setattr(torch.ops.mindiesd, "quant_flash_attn", execute, raising=False)
monkeypatch.setattr(torch.ops.mindiesd, "quant_flash_attn_metadata", metadata, raising=False)
query = torch.randn(batch, q_len, q_heads, 64, dtype=torch.bfloat16)
key = torch.randn(batch, kv_len, 2, 64, dtype=torch.bfloat16)
value = torch.randn_like(key)
if layout == "BNSD":
query, key, value = (tensor.transpose(1, 2) for tensor in (query, key, value))
impl = flash_attn.FlashAttentionImpl(
num_heads=q_heads,
head_size=64,
softmax_scale=0.37,
qkv_layout=layout,
)
output = impl.forward_npu(query, key, value, AttentionMetadata(extra={"kv_cache_dtype": precision}))
rotation = get_quant_attention_rotation(query.device, query.dtype, 64)
expected = query @ rotation
torch.testing.assert_close(output, expected)
assert output.shape == query.shape and len(quantized_inputs) == 3
execute.assert_called_once()
kwargs = execute.call_args.kwargs
assert kwargs["softmax_scale"] == 0.37
if precision == "fp8":
assert kwargs["input_layout"] == "BNSD"
assert quantized_inputs[0][0].shape == (2, 130, 64)
assert quantized_inputs[0][1]["row_block_size"] == 128
elif precision == "mxfp8":
args = execute.call_args.args
assert args[0].shape == (batch * q_len, q_heads, 64)
assert args[1].shape == args[2].shape == (batch * kv_len, 2, 64)
assert kwargs["input_layout"] == "TND"
assert kwargs["num_query_heads"] == q_heads and kwargs["num_key_value_heads"] == 2
assert kwargs["actual_seq_qlen"] == [q_len * (i + 1) for i in range(batch)]
assert kwargs["actual_seq_kvlen"] == [kv_len * (i + 1) for i in range(batch)]
assert [entry[1]["axis"] for entry in quantized_inputs] == [-1, -1, 0]
else:
seq_axis = 1 if layout == "BSND" else 2
assert quantized_inputs[0][0].shape[seq_axis] == 512
assert kwargs["seqused_q"].tolist() == kwargs["seqused_kv"].tolist() == [512]
assert kwargs["layout_q"] == kwargs["layout_out"] == layout
assert kwargs["q_dtype"] == torch_npu.float4_e2m1fn_x2
assert kwargs["q_descale_dtype"] == torch_npu.float8_e8m0fnu
assert execute.call_args.args[3].shape[:2] == (1, 2)
assert [entry[1]["axis"] for entry in quantized_inputs] == [-1, -1, seq_axis]
assert kwargs["metadata"] is metadata.return_value
metadata.assert_called_once()
@pytest.mark.parametrize("precision", ["fp8", "mxfp4"])
@pytest.mark.parametrize("length", [65, 600])
def test_omni_calls_real_sparse_public_api(monkeypatch, precision, length):
rf = importlib.import_module("mindiesd.layers.flash_attn.sparse_flash_attn_rf_v3")
monkeypatch.setattr(flash_attn, "current_omni_platform", SimpleNamespace(is_npu=lambda: True, device_name="npu"))
for module in (flash_attn, rainfusion_attn):
monkeypatch.setattr(module, "get_current_diffusion_config_or_none", lambda: None)
rainfusion_attn._mindiesd_supports_precision.cache_clear()
monkeypatch.setattr(rf, "do_tensor_rearrange_only", lambda q, k, v, *a, **kw: (q, k, v))
monkeypatch.setattr(rf, "avgpool", lambda *a, **kw: torch.ones(1))
monkeypatch.setattr(rf, "_generate_mask_direct", lambda *a, **kw: torch.ones(1, 2, 8, 8))
monkeypatch.setattr(rf, "_bsa_inv_rearrange", lambda out, *a: out)
def quantize(tensor, **kwargs):
shape = list(tensor.shape)
axis = kwargs.get("axis", -1)
shape[axis] = max(1, shape[axis] // 32)
return tensor, torch.ones(shape, dtype=torch.uint8)
monkeypatch.setattr(torch_npu, "npu_dynamic_mx_quant", quantize, raising=False)
monkeypatch.setattr(torch_npu, "npu_dynamic_block_quant", quantize, raising=False)
execute = Mock(side_effect=lambda **kw: (torch.ones_like(kw["query"]), None))
monkeypatch.setattr(torch.ops.mindiesd, "block_sparse_attention", execute, raising=False)
query = torch.randn(1, length, 2, 64, dtype=torch.bfloat16)
impl = rainfusion_attn.RainFusionAttentionImpl(
num_heads=2,
head_size=64,
softmax_scale=0.37,
qkv_layout="BSND",
backend_kwargs={"sparsity": 0.8},
)
plan = rainfusion_attn.RainFusionPlan(used_len=length, prefix_len=0, latent_shape=(1, 1, length))
try:
output = impl._forward_sparse_npu(
query,
query,
query,
plan,
AttentionMetadata(extra={"kv_cache_dtype": precision}),
)
finally:
rainfusion_attn._mindiesd_supports_precision.cache_clear()
torch.testing.assert_close(output, torch.ones_like(query))
execute.assert_called_once()
kwargs = execute.call_args.kwargs
assert kwargs["actual_seq_lengths"] == kwargs["actual_seq_lengths_kv"] == [length]
assert kwargs["q_input_layout"] == "BNSD"
assert kwargs["block_sparse_mask"] is not None
if precision == "mxfp4":
assert kwargs["query"].shape[2] == (length + 63) // 64 * 64
assert kwargs["quant_mode"] == 2
assert kwargs["q_dtype"] == torch_npu.float4_e2m1fn_x2
assert kwargs["v_scale_dtype"] == torch_npu.float8_e8m0fnu
@@ -0,0 +1,319 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CPU contract tests for MindIE adapters; no NPU kernels are mocked in production."""
import sys
from types import ModuleType, SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from vllm_omni.diffusion.attention import layer as layer_mod
from vllm_omni.diffusion.attention.backends import flash_attn, rainfusion_attn
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata, VideoTokenLayout
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
_DEFAULT_SKIP_STEPS = frozenset({0, 1, 38, 39})
@pytest.fixture
def runtime(monkeypatch):
mod = ModuleType("mindiesd")
def execute(q, k, v, *, precision, layout, scale, q_rot=None, k_rot=None):
return q
setattr(mod, "quant_attention", Mock(side_effect=execute))
monkeypatch.setitem(sys.modules, mod.__name__, mod)
for module in (flash_attn, layer_mod):
monkeypatch.setattr(module, "current_omni_platform", SimpleNamespace(is_npu=lambda: True, device_name="npu"))
monkeypatch.setattr(flash_attn, "get_current_diffusion_config_or_none", lambda: None)
monkeypatch.setattr(rainfusion_attn, "get_current_diffusion_config_or_none", lambda: None)
monkeypatch.setattr(rainfusion_attn, "is_forward_context_available", lambda: False)
supports_precision = rainfusion_attn._mindiesd_supports_precision
supports_precision.cache_clear()
yield mod
supports_precision.cache_clear()
def flash(layout="BSND"):
return flash_attn.FlashAttentionImpl(
num_heads=2,
head_size=64,
softmax_scale=0.37,
qkv_layout=layout,
)
def sparse(**kwargs):
return rainfusion_attn.RainFusionAttentionImpl(
num_heads=2,
head_size=64,
softmax_scale=0.37,
qkv_layout="BSND",
backend_kwargs={"sparsity": 0.8, **kwargs},
)
def video_metadata(**extra):
return AttentionMetadata(
video_layout=VideoTokenLayout(prefix_len=0, latent_grid=(4, 32, 32), used_len=4096),
extra={"max_seqlen_q": 4096, **extra},
)
@pytest.mark.parametrize("method", ["fp8", "mxfp8", "mxfp4"])
@pytest.mark.parametrize("layout", [None, "BSND", "BNSD"])
def test_exact_runtime_layout_scale_and_rotation(runtime, method, layout):
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
if layout == "BNSD":
q = q.transpose(1, 2)
impl = flash(layout=layout)
metadata = AttentionMetadata(extra={"kv_cache_dtype": method})
assert impl.forward_npu(q, q, q, metadata) is q
args, kwargs = runtime.quant_attention.call_args
assert args[0] is q # no unconditional transpose or extra quantization
assert kwargs["layout"] == (layout or "BSND") and kwargs["scale"] == 0.37 and kwargs["precision"] == method
assert "attn_mask" not in kwargs
from vllm_omni.platforms.npu.quant.kv_quant_npu import get_quant_attention_rotation
assert kwargs["q_rot"] is kwargs["k_rot"]
torch.testing.assert_close(kwargs["q_rot"], get_quant_attention_rotation(q.device, q.dtype, 64))
def test_missing_runtime_requires_config_change(runtime):
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
del runtime.quant_attention
with pytest.raises(ImportError, match="diffusion_kv_cache_dtype"):
flash().forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": "mxfp4"}))
def test_operator_failure_never_falls_back(runtime):
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
runtime.quant_attention.side_effect = RuntimeError("operator failure")
with pytest.raises(RuntimeError, match="operator failure"):
flash().forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": "mxfp4"}))
runtime.quant_attention.assert_called_once()
@pytest.mark.parametrize("method", ["fp8", "mxfp8", "mxfp4"])
def test_unsupported_shape_requires_config_change(runtime, method):
q = torch.randn(1, 128, 2, 96, dtype=torch.bfloat16)
with pytest.raises(ValueError, match="diffusion_kv_cache_dtype"):
flash().forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": method}))
runtime.quant_attention.assert_not_called()
def test_packed_metadata_never_reaches_quant_runtime(runtime):
q = torch.randn(1, 128, 2, 64, dtype=torch.bfloat16)
metadata = AttentionMetadata(extra={"kv_cache_dtype": "mxfp8", "cu_seqlens_q": torch.tensor([0, 64, 128])})
with pytest.raises(ValueError, match="packed/varlen"):
flash().forward_npu(q, q, q, metadata)
runtime.quant_attention.assert_not_called()
def make_layer(runtime, monkeypatch, method="mxfp8", skip_steps=_DEFAULT_SKIP_STEPS):
layer = object.__new__(layer_mod.Attention)
torch.nn.Module.__init__(layer)
layer.attention = flash()
layer.attn_backend = flash_attn.FlashAttentionBackend
layer._disable_kv_quant = False
layer.layer_idx = 3
cfg = SimpleNamespace(
diffusion_kv_cache_dtype=method,
diffusion_kv_cache_skip_step_indices=skip_steps,
diffusion_kv_cache_skip_layer_indices=None,
parallel_config=SimpleNamespace(ring_degree=1),
)
layer._init_kv_cache_quantization(cfg)
return layer
@pytest.mark.parametrize("method", ["fp8", "mxfp8", "mxfp4"])
def test_global_quant_method_is_published_per_forward(runtime, monkeypatch, method):
layer = make_layer(runtime, monkeypatch, method=method, skip_steps=None)
assert layer._with_kv_cache_dtype(None).extra["kv_cache_dtype"] == method
@pytest.mark.parametrize("method", [None, "auto", "float"])
def test_optout_without_quant_policy_keeps_metadata_unchanged(runtime, monkeypatch, method):
layer = make_layer(runtime, monkeypatch)
layer._disable_kv_quant = True
layer._init_kv_cache_quantization(
SimpleNamespace(
diffusion_kv_cache_dtype=method,
diffusion_kv_cache_skip_step_indices=None,
diffusion_kv_cache_skip_layer_indices=None,
parallel_config=SimpleNamespace(ring_degree=1),
)
)
source = video_metadata()
assert layer._with_kv_cache_dtype(source) is source
def test_step_skip_is_request_local_and_cross_optout(runtime, monkeypatch):
layer = make_layer(runtime, monkeypatch)
ctx = SimpleNamespace(denoise_step_idx=0)
monkeypatch.setattr(layer_mod, "is_forward_context_available", lambda: True)
monkeypatch.setattr(layer_mod, "get_forward_context", lambda: ctx)
source = video_metadata()
for step in list(range(40)) * 2:
ctx.denoise_step_idx = step
resolved = layer._with_kv_cache_dtype(source)
assert resolved.extra["kv_cache_dtype"] == ("float" if step in (0, 1, 38, 39) else "mxfp8")
assert resolved.video_layout is source.video_layout
assert source.extra == {"max_seqlen_q": 4096}
layer._disable_kv_quant = True
assert layer._with_kv_cache_dtype(source).extra["kv_cache_dtype"] == "float"
def test_ring_attention_is_rejected(runtime, monkeypatch):
layer = make_layer(runtime, monkeypatch)
cfg = SimpleNamespace(diffusion_kv_cache_dtype="mxfp8", parallel_config=SimpleNamespace(ring_degree=2))
with pytest.raises(ValueError, match="ring"):
layer._init_kv_cache_quantization(cfg)
def test_sparse_tail_steps_and_dense_quant_dispatch(runtime, monkeypatch):
ctx = SimpleNamespace(denoise_step_idx=49, total_denoise_steps=50)
monkeypatch.setattr(rainfusion_attn, "is_forward_context_available", lambda: True)
monkeypatch.setattr(rainfusion_attn, "get_forward_context", lambda: ctx)
impl = sparse(start_step=2, end_step=2)
metadata = video_metadata(kv_cache_dtype="fp8")
assert impl._resolve_plan(metadata) is None
q = torch.randn(1, 4096, 2, 64, dtype=torch.bfloat16)
impl.forward_npu(q, q, q, metadata)
assert runtime.quant_attention.call_args.kwargs["precision"] == "fp8"
@pytest.mark.parametrize("method", ["fp8", "mxfp4"])
def test_sparse_quant_uses_public_runtime(runtime, monkeypatch, method):
runtime.sparse_attention = Mock(side_effect=lambda q, k, v, **kwargs: q)
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: True)
q = torch.randn(1, 4224, 2, 64, dtype=torch.bfloat16)
out = sparse().forward_npu(q, q, q, video_metadata(kv_cache_dtype=method))
assert out.shape == q.shape and not out[:, 4096:].any()
kwargs = runtime.sparse_attention.call_args.kwargs
assert (kwargs["precision"], kwargs["sparse_type"], kwargs["inner_precise"]) == (method, "rf_v3", 4)
runtime.quant_attention.assert_not_called()
def test_sparse_custom_mask_stays_dense(runtime):
metadata = AttentionMetadata(
attn_mask=torch.ones(1, 4096, dtype=torch.bool),
video_layout=VideoTokenLayout(prefix_len=0, latent_grid=(4, 32, 32)),
extra={"max_seqlen_q": 4096},
)
assert sparse()._resolve_plan(metadata) is None
def test_sparse_model_padding_mask_is_equivalent_to_trimming(runtime):
metadata = video_metadata()
metadata.attn_mask = torch.arange(4224)[None] < 4096
assert sparse()._resolve_plan(metadata).used_len == 4096
def test_bsa_operator_failure_never_retries(runtime, monkeypatch):
runtime.sparse_attention = Mock(side_effect=RuntimeError("BSA operator failure"))
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: True)
q = torch.randn(1, 4096, 2, 64, dtype=torch.bfloat16)
with pytest.raises(RuntimeError, match="BSA operator failure"):
sparse().forward_npu(q, q, q, video_metadata(kv_cache_dtype="mxfp4"))
runtime.sparse_attention.assert_called_once()
@pytest.mark.parametrize(
"method,shape,supports_precision,error",
[
("mxfp8", (1, 4096, 2, 64), True, "sparse mxfp8 is not supported"),
("mxfp4", (1, 4096, 2, 64), False, "explicitly support precision"),
("fp8", (2, 4096, 2, 64), True, "batch size 1"),
("fp8", (1, 4096, 2, 32), True, "power-of-two head dimension"),
("mxfp4", (1, 4096, 2, 96), True, "power-of-two head dimension"),
],
)
def test_invalid_sparse_quant_requires_config_change(runtime, monkeypatch, method, shape, supports_precision, error):
runtime.sparse_attention = Mock(side_effect=lambda q, k, v, **kwargs: q)
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: supports_precision)
q = torch.randn(shape, dtype=torch.bfloat16)
with pytest.raises(ValueError, match=error):
sparse().forward_npu(q, q, q, video_metadata(kv_cache_dtype=method))
runtime.sparse_attention.assert_not_called()
def test_layer_selector_requires_index_except_cross_optout(runtime, monkeypatch):
layer = make_layer(runtime, monkeypatch)
layer.layer_idx = None
config = SimpleNamespace(
diffusion_kv_cache_dtype="mxfp8",
diffusion_kv_cache_skip_step_indices=None,
diffusion_kv_cache_skip_layer_indices={0, 3},
parallel_config=SimpleNamespace(ring_degree=1),
)
with pytest.raises(ValueError, match="parseable transformer block index"):
layer._init_kv_cache_quantization(config)
layer._disable_kv_quant = True
layer._init_kv_cache_quantization(config)
assert layer._with_kv_cache_dtype(None).extra["kv_cache_dtype"] == "float"
@pytest.mark.parametrize("prefix", ["transformer.blocks.3.attn1", "transformer_2.blocks.3.attn1"])
@pytest.mark.parametrize("method", ["fp8", "mxfp4"])
def test_expert_layer_and_step_skip_remain_sparse(runtime, monkeypatch, prefix, method):
def execute(q, k, v, *, precision="bf16", **kwargs):
return q
runtime.sparse_attention = Mock(wraps=execute)
monkeypatch.setattr(rainfusion_attn, "_mindiesd_supports_precision", lambda: True)
layer = make_layer(runtime, monkeypatch)
layer.layer_idx = rainfusion_attn._try_extract_layer_index(prefix)
assert layer.layer_idx == 3
config = SimpleNamespace(
diffusion_kv_cache_dtype=method,
diffusion_kv_cache_skip_step_indices={0, 1, 38, 39},
diffusion_kv_cache_skip_layer_indices={3},
parallel_config=SimpleNamespace(ring_degree=1),
)
impl = sparse()
layer.attention = impl
layer.attn_backend = rainfusion_attn.RainFusionAttentionBackend
layer._init_kv_cache_quantization(config)
ctx = SimpleNamespace(denoise_step_idx=0)
monkeypatch.setattr(layer_mod, "is_forward_context_available", lambda: True)
monkeypatch.setattr(layer_mod, "get_forward_context", lambda: ctx)
source = video_metadata(kv_cache_dtype="mxfp8")
q = torch.randn(1, 4096, 2, 64, dtype=torch.bfloat16)
for layer_idx, step in ((3, 0), (3, 2), (4, 0), (4, 1), (4, 2), (4, 37), (4, 38), (4, 39), (4, 0)):
expected = "bf16" if layer_idx == 3 or step in (0, 1, 38, 39) else method
layer.layer_idx = layer_idx
ctx.denoise_step_idx = step
metadata = layer._with_kv_cache_dtype(source)
impl.forward_npu(q, q, q, metadata)
assert runtime.sparse_attention.call_args.kwargs["precision"] == expected
runtime.quant_attention.assert_not_called()
assert source.extra["kv_cache_dtype"] == "mxfp8"
def test_custom_attention_initialization_keeps_upstream_optout(monkeypatch):
monkeypatch.setattr(layer_mod, "get_current_diffusion_config_or_none", lambda: None)
monkeypatch.setattr(layer_mod, "build_parallel_attention_strategy", lambda **kw: object())
monkeypatch.setattr(layer_mod, "NoParallelAttention", lambda: object())
custom = torch.nn.Identity()
layer = layer_mod.Attention(
num_heads=2,
head_size=64,
softmax_scale=0.125,
causal=False,
custom_attention=custom,
skip_sequence_parallel=True,
)
assert layer.attention is custom and layer._kv_cache_dtype is None
def test_legacy_global_fp8_uses_implicit_bsnd(runtime):
q = torch.randn(1, 17, 2, 64, dtype=torch.bfloat16)
impl = flash(layout=None)
assert impl.forward_npu(q, q, q, AttentionMetadata(extra={"kv_cache_dtype": "fp8"})) is q
assert runtime.quant_attention.call_args.kwargs["layout"] == "BSND"
assert runtime.quant_attention.call_args.kwargs["precision"] == "fp8"
@@ -133,18 +133,17 @@ def test_invalid_ref2va_video_spans_fall_back_to_dense():
assert make_impl()._resolve_plan(AttentionMetadata(extra={"max_seqlen_q": 12000}, video_layout=layout)) is None
def test_validate_available_rejects_legacy_mindiesd(monkeypatch):
def test_validate_available_accepts_single_video_mindiesd(monkeypatch):
mindiesd = types.ModuleType("mindiesd")
def sparse_attention(query, key, value, **kwargs):
return query
mindiesd.sparse_attention = sparse_attention
monkeypatch.setattr(mindiesd, "sparse_attention", sparse_attention, raising=False)
monkeypatch.setitem(sys.modules, "mindiesd", mindiesd)
monkeypatch.setattr("importlib.util.find_spec", lambda _: object())
with pytest.raises(ValueError, match="video_spans"):
RainFusionAttentionBackend.validate_available()
RainFusionAttentionBackend.validate_available()
def test_validate_available_accepts_new_mindiesd(monkeypatch):
@@ -153,7 +152,7 @@ def test_validate_available_accepts_new_mindiesd(monkeypatch):
def sparse_attention(query, key, value, *, video_spans=None, **kwargs):
return query
mindiesd.sparse_attention = sparse_attention
monkeypatch.setattr(mindiesd, "sparse_attention", sparse_attention, raising=False)
monkeypatch.setitem(sys.modules, "mindiesd", mindiesd)
monkeypatch.setattr("importlib.util.find_spec", lambda _: object())
@@ -300,22 +299,21 @@ def _fake_mindiesd_module():
import types
fake = types.ModuleType("mindiesd")
fake.sparse_attention = lambda *args, **kwargs: None
setattr(fake, "sparse_attention", lambda *args, **kwargs: None)
return fake
def test_precision_non_bf16_requires_mindiesd_support():
"""precision != bf16 against a mindiesd lacking the kwarg must raise RuntimeError."""
"""Unsupported sparse precision must fail before any operator executes."""
import sys
impl = make_impl(precision="mix")
sys.modules["mindiesd"] = _fake_mindiesd_module()
try:
with mock.patch.dict(sys.modules, {"mindiesd": _fake_mindiesd_module()}):
with mock.patch.object(rainfusion_attn, "_mindiesd_supports_precision", return_value=False):
with pytest.raises(RuntimeError, match="requires MindIE-SD"):
impl._forward_sparse_npu(None, None, None, None)
finally:
sys.modules.pop("mindiesd", None)
with pytest.raises(ValueError, match="explicitly support precision"):
impl._forward_sparse_npu(
None, None, None, RainFusionPlan(prefix_len=0, used_len=8, latent_shape=[2, 2, 2])
)
def test_precision_non_bf16_passes_gate_when_supported():
@@ -323,8 +321,7 @@ def test_precision_non_bf16_passes_gate_when_supported():
import sys
impl = make_impl(precision="mix")
sys.modules["mindiesd"] = _fake_mindiesd_module()
try:
with mock.patch.dict(sys.modules, {"mindiesd": _fake_mindiesd_module()}):
with mock.patch.object(rainfusion_attn, "_mindiesd_supports_precision", return_value=True):
# q/k/v shapes: [B, S, N, D]; plan geometry must match S.
q = torch.randn(1, 8, 4, 128)
@@ -332,5 +329,3 @@ def test_precision_non_bf16_passes_gate_when_supported():
# The gate must pass; the fake mindiesd returns None so no crash.
out = impl._forward_sparse_npu(q, q, q, plan)
assert out is None
finally:
sys.modules.pop("mindiesd", None)
@@ -0,0 +1,40 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Wan integration contracts required by quantized dense/sparse attention."""
from unittest.mock import Mock
import pytest
import torch
from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformerBlock
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
def test_block_publishes_unpadded_grid_without_partial_packed_metadata():
block = object.__new__(WanTransformerBlock)
torch.nn.Module.__init__(block)
block.scale_shift_table = torch.zeros(1, 6, 4)
block.norm1 = lambda x, *a: x
block.norm2 = lambda x: x
block.norm3 = lambda x, *a: x
block.attn1 = Mock(side_effect=lambda x, *a: torch.zeros_like(x))
block.attn2 = Mock(side_effect=lambda x, *a: torch.zeros_like(x))
block.ffn = lambda x: torch.zeros_like(x)
# Local SP tokens include alignment padding; grid describes the full valid video.
x = torch.zeros(1, 1152, 4)
block(
x,
x,
torch.zeros(1, 6, 4),
None,
hidden_states_mask=torch.ones(1, 1152, dtype=torch.bool),
vsa_dit_seq_shape=(5, 16, 27),
)
metadata = block.attn1.call_args.args[2]
assert metadata.video_layout.latent_grid == (5, 16, 27)
assert metadata.video_layout.used_len == 2160
assert metadata.video_layout.prefix_len == 0
assert "max_seqlen_q" not in metadata.extra # CUDA must not infer incomplete packed varlen.
assert block.attn2.call_args.args[2] is None
+35 -145
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for NPU FP8 KV quantization helpers.
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Tests for seeded rotations and the quantized NPU attention backend.
These tests load ``kv_quant_npu`` from its source file via ``importlib`` so
the test module itself does not ``import vllm_omni`` (which would pull
@@ -10,10 +10,8 @@ the test module itself does not ``import vllm_omni`` (which would pull
from __future__ import annotations
import importlib.util
import math
from pathlib import Path
from types import ModuleType, SimpleNamespace
from typing import Any
from types import ModuleType
import pytest
import torch
@@ -61,147 +59,39 @@ def _npu_smoke_available() -> bool:
npu_smoke = pytest.mark.skipif(not _npu_smoke_available(), reason="NPU device or torch_npu not available.")
def test_is_quantized_kv_cache() -> None:
assert kv_quant_npu.is_quantized_kv_cache("fp8")
assert not kv_quant_npu.is_quantized_kv_cache(None)
assert not kv_quant_npu.is_quantized_kv_cache("int8")
class TestKVQuantNPUUnit:
@pytest.fixture(autouse=True)
def clear_rot_cache(self):
kv_quant_npu._ROT_MATRIXS.clear()
def test_get_rot_matrix_caches_by_device_dtype_and_head_dim(self) -> None:
calls = {"count": 0}
class FakeQuaRotMode:
HADAMARD = "hadamard"
def fake_create_rot(mode, head_dim, seed):
calls["count"] += 1
assert mode == FakeQuaRotMode.HADAMARD
assert seed == 425500
return torch.eye(head_dim, dtype=torch.float32)
device = torch.device("cpu")
rot_1 = kv_quant_npu._get_rot_matrix(device, torch.float16, 8, FakeQuaRotMode, fake_create_rot)
rot_2 = kv_quant_npu._get_rot_matrix(device, torch.float16, 8, FakeQuaRotMode, fake_create_rot)
rot_3 = kv_quant_npu._get_rot_matrix(device, torch.bfloat16, 8, FakeQuaRotMode, fake_create_rot)
rot_4 = kv_quant_npu._get_rot_matrix(device, torch.float16, 16, FakeQuaRotMode, fake_create_rot)
assert calls["count"] == 3
assert rot_1 is rot_2
assert rot_3.dtype == torch.bfloat16
assert rot_4.shape == (16, 16)
@pytest.fixture
def fake_quant_ops(self, monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
captured: dict[str, Any] = {
"fa_calls": [],
"npu_kwargs": None,
"out_shape": None,
}
class FakeTorchNPU:
float8_e4m3fn = "fp8_marker"
@staticmethod
def npu_fused_infer_attention_score_v2(q, k, v, **kwargs):
del q, k, v
captured["npu_kwargs"] = kwargs
out_shape = captured["out_shape"]
return (torch.ones(out_shape, dtype=torch.float32),)
def fake_fa_block_quant_preprocess(x, block_size, dst_type, layout):
captured["fa_calls"].append(
{
"block_size": block_size,
"layout": layout,
"dst_type": dst_type,
"shape": tuple(x.shape),
}
)
scale = torch.full((1,), float(block_size), dtype=torch.float32)
return x, scale
fake_qua_rot_mode = SimpleNamespace(HADAMARD="hadamard")
def fake_create_rot(mode, head_dim, seed):
assert mode == "hadamard"
assert seed == 425500
return torch.eye(head_dim, dtype=torch.float32)
monkeypatch.setattr(
kv_quant_npu,
"_load_quant_ops",
lambda: (FakeTorchNPU, fake_fa_block_quant_preprocess, fake_qua_rot_mode, fake_create_rot),
)
return captured
@staticmethod
def _make_qkv(shape: tuple[int, int, int, int]) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
query = torch.randn(*shape, dtype=torch.float32)
key = torch.randn(*shape, dtype=torch.float32)
value = torch.randn(*shape, dtype=torch.float32)
return query, key, value
@pytest.mark.parametrize(
"layout,input_shape,out_shape,softmax_scale,expected_scale",
[
("BNSD", (2, 3, 4, 8), (2, 3, 6, 8), None, 1.0 / math.sqrt(8)),
("BSND", (2, 4, 3, 8), (2, 6, 3, 8), 0.125, 0.125),
],
)
def test_fp8_rotate_quant_fa_layouts_scale_and_crop(
self,
fake_quant_ops: dict[str, Any],
layout: str,
input_shape: tuple[int, int, int, int],
out_shape: tuple[int, int, int, int],
softmax_scale: float | None,
expected_scale: float,
) -> None:
query, key, value = self._make_qkv(input_shape)
fake_quant_ops["out_shape"] = out_shape
out = kv_quant_npu.fp8_rotate_quant_fa(query, key, value, layout=layout, softmax_scale=softmax_scale)
assert out.shape == query.shape
assert out.dtype == query.dtype
assert fake_quant_ops["npu_kwargs"]["input_layout"] == layout
# BNSD: shape[1]==heads, BSND: shape[2]==heads.
expected_heads = input_shape[1] if layout == "BNSD" else input_shape[2]
assert fake_quant_ops["npu_kwargs"]["num_query_heads"] == expected_heads
assert fake_quant_ops["npu_kwargs"]["softmax_scale"] == pytest.approx(expected_scale)
assert [call["block_size"] for call in fake_quant_ops["fa_calls"]] == [128, 256, 256]
def test_fp8_rotate_quant_fa_invalid_layout_raises(self, fake_quant_ops) -> None:
query = torch.randn(1, 2, 3, 4, dtype=torch.float32)
key = torch.randn(1, 2, 3, 4, dtype=torch.float32)
value = torch.randn(1, 2, 3, 4, dtype=torch.float32)
fake_quant_ops["out_shape"] = (1, 2, 3, 4)
with pytest.raises(ValueError, match="unsupported layout"):
kv_quant_npu.fp8_rotate_quant_fa(query, key, value, layout="INVALID")
def test_rotation_is_fixed_orthogonal_cached_and_preserves_rng():
kv_quant_npu.get_quant_attention_rotation.cache_clear()
before = torch.random.get_rng_state()
rot = kv_quant_npu.get_quant_attention_rotation(torch.device("cpu"), torch.float32, 64)
assert rot is kv_quant_npu.get_quant_attention_rotation(torch.device("cpu"), torch.float32, 64)
torch.testing.assert_close(rot @ rot.T, torch.eye(64))
assert torch.equal(before, torch.random.get_rng_state())
@npu_smoke
class TestKVQuantNPUSmoke:
"""Smoke tests using real torch_npu/mindiesd stack, only on NPU."""
@pytest.mark.npu
@pytest.mark.parametrize("method", ["fp8", "mxfp8", "mxfp4"])
@pytest.mark.parametrize("layout", ["BSND", "BNSD"])
@pytest.mark.parametrize("seq_len", [256, 75600])
def test_dense_runtime_backend_real_npu(method, layout, seq_len):
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionImpl
def test_fp8_rotate_quant_fa_real_npu_shape_contract(self):
try:
kv_quant_npu._load_quant_ops.cache_clear()
kv_quant_npu._load_quant_ops()
except ImportError:
pytest.skip("NPU quant dependencies are not fully installed.")
query = torch.randn(1, 2, 4, 64, dtype=torch.float16, device="npu")
key = torch.randn(1, 2, 4, 64, dtype=torch.float16, device="npu")
value = torch.randn(1, 2, 4, 64, dtype=torch.float16, device="npu")
out = kv_quant_npu.fp8_rotate_quant_fa(query, key, value, layout="BNSD")
assert out.shape == query.shape
assert out.dtype == query.dtype
# On NPU, missing Runtime symbols must fail this qualification test.
query = torch.randn(1, seq_len, 2, 128, dtype=torch.bfloat16, device="npu") * 0.1
# Distinct constant values per head detect head/layout mixing and tail writes.
value = torch.tensor([0.5, -0.5], dtype=query.dtype, device=query.device)
value = value.view(1, 1, 2, 1).expand_as(query).contiguous()
if layout == "BNSD":
query, value = query.transpose(1, 2), value.transpose(1, 2)
impl = FlashAttentionImpl(
num_heads=2,
head_size=128,
softmax_scale=128**-0.5,
qkv_layout=layout,
)
metadata = AttentionMetadata(extra={"kv_cache_dtype": method})
out = impl.forward_npu(query, query, value, metadata)
assert out.shape == query.shape and out.dtype == query.dtype
assert torch.isfinite(out).all()
torch.testing.assert_close(out.float().cpu(), value.float().cpu(), rtol=0.02, atol=0.02)