[NPU][Diffusion] MiniMax-H3 RainFusion end_step tail fallback (#6037)
Signed-off-by: HAAZZZEEEE <wang-taicheng@qq.com>
This commit is contained in:
1 parent
a5b8c82788
commit
c50b7de167
8 files changed
+243
-14
No files matched your search
@@ -101,7 +101,12 @@ class TestAttentionSpec:
|
||||
def test_block_sparse_defaults_applied_when_backend_selected(self):
|
||||
spec = AttentionSpec(backend="RAINFUSION_ATTN")
|
||||
assert spec.block_sparse.sparsity == 0.8
|
||||
assert spec.backend_kwargs() == {"sparsity": 0.8, "start_step": 0}
|
||||
assert spec.backend_kwargs() == {
|
||||
"sparsity": 0.8,
|
||||
"start_step": 0,
|
||||
"end_step": 0,
|
||||
"precision": "bf16",
|
||||
}
|
||||
|
||||
def test_block_sparse_skip_layers_selector_expanded(self):
|
||||
spec = AttentionSpec(
|
||||
@@ -112,6 +117,8 @@ class TestAttentionSpec:
|
||||
assert spec.backend_kwargs() == {
|
||||
"sparsity": 0.9,
|
||||
"start_step": 12,
|
||||
"end_step": 0,
|
||||
"precision": "bf16",
|
||||
"skip_layers": [0, 1, 2, 38],
|
||||
}
|
||||
|
||||
@@ -121,12 +128,44 @@ class TestAttentionSpec:
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"block_sparse",
|
||||
[{"sparsity": 1.5}, {"start_step": -1}],
|
||||
[{"sparsity": 1.5}, {"start_step": -1}, {"end_step": -1}, {"precision": "bogus"}],
|
||||
)
|
||||
def test_block_sparse_invalid_values(self, block_sparse):
|
||||
with pytest.raises(ValueError):
|
||||
AttentionSpec(backend="RAINFUSION_ATTN", block_sparse=block_sparse)
|
||||
|
||||
def test_block_sparse_end_step_precision_forwarded(self):
|
||||
spec = AttentionSpec(
|
||||
backend="RAINFUSION_ATTN",
|
||||
block_sparse={"sparsity": 0.9, "start_step": 5, "end_step": 3, "precision": "fp8"},
|
||||
)
|
||||
assert spec.backend_kwargs() == {
|
||||
"sparsity": 0.9,
|
||||
"start_step": 5,
|
||||
"end_step": 3,
|
||||
"precision": "fp8",
|
||||
}
|
||||
|
||||
def test_block_sparse_precision_enum_input_normalized(self):
|
||||
"""RainFusionPrecision enum members must normalize to their string value.
|
||||
|
||||
Regression: the str,Enum members passed the value check but str() later
|
||||
produced "RainFusionPrecision.MIX" instead of "mix" (StrEnum fixes it).
|
||||
"""
|
||||
from vllm_omni.diffusion.attention.backends.rainfusion_attn import RainFusionConfig
|
||||
from vllm_omni.diffusion.data import RainFusionPrecision
|
||||
|
||||
spec = AttentionSpec(
|
||||
backend="RAINFUSION_ATTN",
|
||||
block_sparse={"sparsity": 0.8, "precision": RainFusionPrecision.MIX},
|
||||
)
|
||||
kwargs = spec.backend_kwargs()
|
||||
assert kwargs["precision"] == "mix"
|
||||
# The config path is where str(value) used to mangle enum members.
|
||||
cfg = RainFusionConfig.from_backend_kwargs(kwargs)
|
||||
assert cfg.precision == "mix"
|
||||
assert type(cfg.precision) is str
|
||||
|
||||
|
||||
class TestAttentionConfig:
|
||||
def test_empty_config(self):
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Geometry contract RAINFUSION_ATTN enforces before handing a forward to rf_v2.
|
||||
|
||||
@@ -13,10 +13,12 @@ to the always-kept prefix segment before generating the block mask.
|
||||
import dataclasses
|
||||
import sys
|
||||
import types
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.attention.backends import rainfusion_attn
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata, VideoTokenLayout, VideoTokenSpan
|
||||
from vllm_omni.diffusion.attention.backends.rainfusion_attn import (
|
||||
_BLOCK_SIZE,
|
||||
@@ -26,6 +28,8 @@ from vllm_omni.diffusion.attention.backends.rainfusion_attn import (
|
||||
)
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.npu]
|
||||
|
||||
PREFIX_ROWS = 710 # 14 text rows + 696 audio rows
|
||||
ALIGNED_GRID = (62, 24, 40) # 1280x768 -> 59520 video rows, 465 blocks
|
||||
MISALIGNED_GRID = (62, 24, 42) # 1344x768 -> 62496 video rows, 488 blocks + 32 rows
|
||||
@@ -228,7 +232,9 @@ def test_irregular_video_tail_reproduces_dense_attention_with_updated_mindiesd()
|
||||
heads, head_dim = 4, 128
|
||||
q, k, v = (torch.randn(1, used, heads, head_dim, dtype=torch.bfloat16, device="npu") for _ in range(3))
|
||||
|
||||
impl = make_impl()
|
||||
# Explicit bf16: the dense comparison exercises the plain sparse kernel,
|
||||
# and older MindIE-SD releases do not accept precision= (gate would raise).
|
||||
impl = make_impl(precision="bf16")
|
||||
impl.rainfusion = dataclasses.replace(impl.rainfusion, sparsity=0.0)
|
||||
plan = RainFusionPlan(prefix_len=prefix_len, used_len=used, latent_shape=list(grid))
|
||||
out = impl._forward_sparse_npu(q, k, v, plan)
|
||||
@@ -241,3 +247,90 @@ def test_irregular_video_tail_reproduces_dense_attention_with_updated_mindiesd()
|
||||
).transpose(1, 2)
|
||||
error = (out.float() - reference.float()).abs().mean() / reference.float().abs().mean()
|
||||
assert error < 2e-3, f"mean relative error {error:.4%} against dense attention"
|
||||
|
||||
|
||||
# end_step tail fallback: the last ``end_step`` denoise steps must stay dense.
|
||||
|
||||
|
||||
def test_end_step_tail_window_stays_dense():
|
||||
"""step_idx inside the last end_step window must resolve no sparse plan."""
|
||||
impl = make_impl(end_step=3)
|
||||
fc = mock.Mock()
|
||||
fc.denoise_step_idx = 47
|
||||
fc.total_denoise_steps = 50
|
||||
with (
|
||||
mock.patch.object(rainfusion_attn, "is_forward_context_available", return_value=True),
|
||||
mock.patch.object(rainfusion_attn, "get_forward_context", return_value=fc),
|
||||
):
|
||||
assert impl._resolve_plan(make_metadata(ALIGNED_GRID)) is None
|
||||
|
||||
|
||||
def test_end_step_before_tail_window_still_sparse():
|
||||
"""step_idx before the tail window must still resolve a sparse plan."""
|
||||
impl = make_impl(end_step=3)
|
||||
fc = mock.Mock()
|
||||
fc.denoise_step_idx = 46
|
||||
fc.total_denoise_steps = 50
|
||||
with (
|
||||
mock.patch.object(rainfusion_attn, "is_forward_context_available", return_value=True),
|
||||
mock.patch.object(rainfusion_attn, "get_forward_context", return_value=fc),
|
||||
):
|
||||
assert impl._resolve_plan(make_metadata(ALIGNED_GRID)) is not None
|
||||
|
||||
|
||||
def test_end_step_zero_never_triggers_tail_fallback():
|
||||
"""end_step=0 must not fall back even on the final denoise step."""
|
||||
impl = make_impl(end_step=0)
|
||||
fc = mock.Mock()
|
||||
fc.denoise_step_idx = 49
|
||||
fc.total_denoise_steps = 50
|
||||
with (
|
||||
mock.patch.object(rainfusion_attn, "is_forward_context_available", return_value=True),
|
||||
mock.patch.object(rainfusion_attn, "get_forward_context", return_value=fc),
|
||||
):
|
||||
assert impl._resolve_plan(make_metadata(ALIGNED_GRID)) is not None
|
||||
|
||||
|
||||
# precision capability gate: a mindiesd without sparse_attention(precision=)
|
||||
# must fail fast instead of silently running the BF16 path.
|
||||
|
||||
|
||||
def _fake_mindiesd_module():
|
||||
"""Install a minimal stand-in so the import inside _forward_sparse_npu succeeds."""
|
||||
import types
|
||||
|
||||
fake = types.ModuleType("mindiesd")
|
||||
fake.sparse_attention = lambda *args, **kwargs: None
|
||||
return fake
|
||||
|
||||
|
||||
def test_precision_non_bf16_requires_mindiesd_support():
|
||||
"""precision != bf16 against a mindiesd lacking the kwarg must raise RuntimeError."""
|
||||
import sys
|
||||
|
||||
impl = make_impl(precision="mix")
|
||||
sys.modules["mindiesd"] = _fake_mindiesd_module()
|
||||
try:
|
||||
with mock.patch.object(rainfusion_attn, "_mindiesd_supports_precision", return_value=False):
|
||||
with pytest.raises(RuntimeError, match="requires MindIE-SD"):
|
||||
impl._forward_sparse_npu(None, None, None, None)
|
||||
finally:
|
||||
sys.modules.pop("mindiesd", None)
|
||||
|
||||
|
||||
def test_precision_non_bf16_passes_gate_when_supported():
|
||||
"""precision != bf16 with a capable mindiesd must not raise from the gate."""
|
||||
import sys
|
||||
|
||||
impl = make_impl(precision="mix")
|
||||
sys.modules["mindiesd"] = _fake_mindiesd_module()
|
||||
try:
|
||||
with mock.patch.object(rainfusion_attn, "_mindiesd_supports_precision", return_value=True):
|
||||
# q/k/v shapes: [B, S, N, D]; plan geometry must match S.
|
||||
q = torch.randn(1, 8, 4, 128)
|
||||
plan = RainFusionPlan(prefix_len=0, used_len=8, latent_shape=[2, 2, 2])
|
||||
# The gate must pass; the fake mindiesd returns None so no crash.
|
||||
out = impl._forward_sparse_npu(q, q, q, plan)
|
||||
assert out is None
|
||||
finally:
|
||||
sys.modules.pop("mindiesd", None)
|
||||
Reference in new issue
Block a user