[Feat] Minimax H3 support ref and tgt video sparse attention on NPU (#6518)

Signed-off-by: fan2956 <zhoufan53@huawei.com>
Co-authored-by: WeiQing Chen <40507679+david6666666@users.noreply.github.com>
This commit is contained in:
fan2956andWeiQing Chen authored and GitHub committed 2026-08-29 11:10:53 +08:00
1 parent d94061201d
commit 0dc8f45906
7 files changed
+312 -32

No files matched your search

@@ -74,6 +74,8 @@ def test_t2va_and_fl2va_packing_keep_update_rows_separate():
assert fl2va["update_mask"].sum().item() == 12
assert (~fl2va["update_mask"]).sum().item() == 6
assert t2va["audio_pos"].numel() == fl2va["audio_pos"].numel() == 6
assert t2va["video_spans"] == ({"start": 10, "latent_grid": (2, 2, 3), "role": "target"},)
assert fl2va["video_spans"] == ({"start": 16, "latent_grid": (2, 2, 3), "role": "target"},)
def test_ref2va_packing_tracks_video_and_audio_update_masks():
@@ -101,6 +103,33 @@ def test_ref2va_packing_tracks_video_and_audio_update_masks():
assert packed["cu_seqlens"].tolist() == [0, 30, 64]
def test_ref2va_packing_publishes_physical_video_spans_only():
from vllm_omni.diffusion.models.minimax_h3.packed_sequence import (
minimax_h3_packed_sequence_ref2va_blocks,
)
packed = minimax_h3_packed_sequence_ref2va_blocks(
text_len=4,
latent_t=2,
latent_h=4,
latent_w=6,
audio_t=3,
ref_blocks=[
{"kind": "image", "latent_h": 4, "latent_w": 4},
{"kind": "video_audio", "ref_audio_t": 2, "latent_t": 2, "latent_h": 4, "latent_w": 6},
{"kind": "audio", "ref_audio_t": 1},
{"kind": "video", "ref_audio_t": 0, "latent_t": 1, "latent_h": 4, "latent_w": 4},
],
)
# Image and all audio rows intentionally stay out of the sparse spans.
assert packed["video_spans"] == (
{"start": 12, "latent_grid": (2, 2, 3), "role": "reference"},
{"start": 26, "latent_grid": (1, 2, 2), "role": "reference"},
{"start": 36, "latent_grid": (2, 2, 3), "role": "target"},
)
def test_condition_noise_is_seeded_and_keeps_clean_anchor_at_timestep_one():
from vllm_omni.diffusion.models.minimax_h3.condition_noise import (
minimax_h3_audio_cond_noise_aug_rows,