Merge branch 'main' into perf/minimax-h3-lossless-hot-path

This commit is contained in:
WeiQing Chen authored and GitHub committed 2026-08-07 11:27:30 +08:00
commit 5ed153947b
143 files changed
+8587 -2165

No files matched your search

@@ -90,6 +90,35 @@ class TestAttentionSpec:
with pytest.raises(ValueError, match=match):
AttentionSpec(**spec)
def test_block_sparse_defaults_applied_when_backend_selected(self):
spec = AttentionSpec(backend="RAINFUSION_ATTN")
assert spec.block_sparse.sparsity == 0.8
assert spec.backend_kwargs() == {"sparsity": 0.8, "start_step": 0}
def test_block_sparse_skip_layers_selector_expanded(self):
spec = AttentionSpec(
backend="RAINFUSION_ATTN",
block_sparse={"sparsity": 0.9, "start_step": 12, "skip_layers": "0-2,38"},
)
assert spec.block_sparse.skip_layer_indices == {0, 1, 2, 38}
assert spec.backend_kwargs() == {
"sparsity": 0.9,
"start_step": 12,
"skip_layers": [0, 1, 2, 38],
}
def test_block_sparse_rejected_on_dense_backend(self):
with pytest.raises(ValueError, match="block_sparse is only supported by"):
AttentionSpec(backend="FLASH_ATTN", block_sparse={"sparsity": 0.8})
@pytest.mark.parametrize(
"block_sparse",
[{"sparsity": 1.5}, {"start_step": -1}],
)
def test_block_sparse_invalid_values(self, block_sparse):
with pytest.raises(ValueError):
AttentionSpec(backend="RAINFUSION_ATTN", block_sparse=block_sparse)
class TestAttentionConfig:
def test_empty_config(self):
@@ -0,0 +1,64 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import nullcontext
import pytest
import torch
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.backends.cudnn_attn import CuDNNAttentionImpl
pytestmark = [pytest.mark.diffusion, pytest.mark.cpu, pytest.mark.core_model]
def test_cudnn_slices_valid_kv_prefix_without_padding_mask(monkeypatch):
observed = {}
def fake_sdpa(query, key, value, **kwargs):
observed.update(query=query, key=key, value=value, kwargs=kwargs)
return query
monkeypatch.setattr(
"vllm_omni.diffusion.attention.backends.cudnn_attn.sdpa_kernel",
lambda _backends: nullcontext(),
)
monkeypatch.setattr(torch.nn.functional, "scaled_dot_product_attention", fake_sdpa)
impl = CuDNNAttentionImpl(
num_heads=2,
head_size=4,
softmax_scale=0.5,
)
query = torch.randn(1, 8, 2, 4)
key = torch.randn_like(query)
value = torch.randn_like(query)
output = impl.forward_cuda(
query,
key,
value,
AttentionMetadata(extra={"valid_kv_length": 5}),
)
assert output.shape == query.shape
assert observed["query"].shape == (1, 2, 8, 4)
assert observed["key"].shape == (1, 2, 5, 4)
assert observed["value"].shape == (1, 2, 5, 4)
assert observed["kwargs"]["attn_mask"] is None
def test_cudnn_rejects_invalid_valid_kv_length():
impl = CuDNNAttentionImpl(
num_heads=2,
head_size=4,
softmax_scale=0.5,
)
query = torch.randn(1, 8, 2, 4)
with pytest.raises(ValueError, match="valid_kv_length"):
impl.forward_cuda(
query,
query,
query,
AttentionMetadata(extra={"valid_kv_length": 9}),
)
@@ -0,0 +1,157 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Geometry contract RAINFUSION_ATTN enforces before handing a forward to rf_v2.
The grids here are the ones MiniMax-H3 FL2VA actually produces for an 8.7s clip:
1280x768 gives a 62x24x40 latent grid whose 59520 video rows land on the 128-row
kernel block, and 1344x768 gives 62x24x42 whose 62496 rows straddle it. Only the
first can run sparse: rf_v2 tiles the sequence as a whole but pools its block
mask from the video and prefix segments separately, so the two only agree on the
block count when the video segment ends on a boundary.
"""
import dataclasses
import pytest
import torch
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata, VideoTokenLayout
from vllm_omni.diffusion.attention.backends.rainfusion_attn import (
_BLOCK_SIZE,
RainFusionAttentionImpl,
RainFusionPlan,
)
from vllm_omni.platforms import current_omni_platform
PREFIX_ROWS = 710 # 14 text rows + 696 audio rows
ALIGNED_GRID = (62, 24, 40) # 1280x768 -> 59520 video rows, 465 blocks
MISALIGNED_GRID = (62, 24, 42) # 1344x768 -> 62496 video rows, 488 blocks + 32 rows
def make_impl(**backend_kwargs):
return RainFusionAttentionImpl(
num_heads=8,
head_size=128,
softmax_scale=128**-0.5,
prefix="transformer_blocks.0.attn",
qkv_layout="BSND",
backend_kwargs={"sparsity": 0.8, **backend_kwargs},
)
def make_metadata(grid, prefix_len=PREFIX_ROWS, max_seqlen_q=None):
video_rows = grid[0] * grid[1] * grid[2]
return AttentionMetadata(
extra={"max_seqlen_q": prefix_len + video_rows if max_seqlen_q is None else max_seqlen_q},
video_layout=VideoTokenLayout(prefix_len=prefix_len, latent_grid=grid),
)
def test_block_aligned_video_segment_runs_sparse():
plan = make_impl()._resolve_plan(make_metadata(ALIGNED_GRID))
assert plan is not None
assert plan.prefix_len == PREFIX_ROWS
assert plan.used_len == PREFIX_ROWS + 59520
assert plan.latent_shape == list(ALIGNED_GRID)
def test_misaligned_video_segment_stays_dense():
# Realigning the sequence would mean padding it, and rf_v2 ignores attn_mask:
# pad keys would take a share of every real query's softmax denominator.
assert make_impl()._resolve_plan(make_metadata(MISALIGNED_GRID)) is None
@pytest.mark.parametrize("prefix_len", [1, 127, _BLOCK_SIZE, 710, 900])
def test_prefix_length_does_not_affect_alignment(prefix_len):
# Only the video segment has to land on a block boundary; rf_v2 pools the
# prefix separately and keeps every one of its blocks.
plan = make_impl()._resolve_plan(make_metadata(ALIGNED_GRID, prefix_len=prefix_len))
assert plan is not None
assert plan.prefix_len == prefix_len
def test_sparsity_zero_never_resolves_a_plan():
assert make_impl(sparsity=0.0)._resolve_plan(make_metadata(ALIGNED_GRID)) is None
def test_missing_video_layout_falls_back_to_dense():
assert make_impl()._resolve_plan(AttentionMetadata(extra={"max_seqlen_q": 60230})) is None
def test_missing_max_seqlen_falls_back_to_dense():
metadata = AttentionMetadata(
video_layout=VideoTokenLayout(prefix_len=PREFIX_ROWS, latent_grid=ALIGNED_GRID),
)
assert make_impl()._resolve_plan(metadata) is None
def test_video_segment_must_be_the_tail_of_packed_document_zero():
metadata = make_metadata(ALIGNED_GRID, max_seqlen_q=PREFIX_ROWS + 59520 + 128)
assert make_impl()._resolve_plan(metadata) is None
@pytest.mark.parametrize("grid", [(4, 24, 40), (1, 24, 40)])
def test_short_video_stays_dense(grid):
assert make_impl()._resolve_plan(make_metadata(grid)) is None
@pytest.mark.parametrize("qkv_layout", ["BNSD", "BSH"])
def test_explicitly_wrong_layout_is_rejected(qkv_layout):
with pytest.raises(ValueError, match="BSND"):
RainFusionAttentionImpl(
num_heads=8,
head_size=128,
softmax_scale=128**-0.5,
qkv_layout=qkv_layout,
backend_kwargs={"sparsity": 0.8},
)
def test_undeclared_layout_stays_dense():
# An undeclared layout is not an error -- the layer keeps working, just
# densely, and the fallback sees the same absent layout plain FLASH_ATTN would.
impl = RainFusionAttentionImpl(
num_heads=8,
head_size=128,
softmax_scale=128**-0.5,
prefix="transformer_blocks.0.attn",
backend_kwargs={"sparsity": 0.8},
)
assert impl._resolve_plan(make_metadata(ALIGNED_GRID)) is None
assert impl.dense_fallback.qkv_layout is None
@pytest.mark.skipif(not current_omni_platform.is_npu(), reason="rf_v2 runs on Ascend NPU only.")
def test_fully_populated_mask_reproduces_dense_attention():
"""At sparsity=0 every key block is kept, so rf_v2 must match dense attention.
This is the guard on the geometry the backend hands the kernel: a wrong
prefix length or latent grid still produces plausible output under sparse
selection, but shows up here as a mismatch against SDPA.
"""
torch.manual_seed(0)
prefix_len, grid = 384, (4, 16, 64) # 4096 video rows = 32 blocks
video_len = grid[0] * grid[1] * grid[2]
used = prefix_len + video_len
heads, head_dim = 4, 128
q, k, v = (torch.randn(1, used, heads, head_dim, dtype=torch.bfloat16, device="npu") for _ in range(3))
impl = make_impl()
impl.rainfusion = dataclasses.replace(impl.rainfusion, sparsity=0.0)
plan = RainFusionPlan(prefix_len=prefix_len, used_len=used, latent_shape=list(grid))
out = impl._forward_sparse_npu(q, k, v, plan)
reference = torch.nn.functional.scaled_dot_product_attention(
q.transpose(1, 2),
k.transpose(1, 2),
v.transpose(1, 2),
scale=impl.softmax_scale,
).transpose(1, 2)
error = (out.float() - reference.float()).abs().mean() / reference.float().abs().mean()
assert error < 2e-3, f"mean relative error {error:.4%} against dense attention"
@@ -8,6 +8,7 @@ import torch
import torch.nn.functional as F
from vllm_omni.diffusion.attention.backends import trtllm_attn as tg
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.backends.trtllm_attn import TrtllmAttentionImpl
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]
@@ -142,6 +143,9 @@ def test_denoise_progress_mixin(monkeypatch):
_Pipe().record_denoise_step(4)
assert ctx.denoise_step_idx == 4 and ctx.denoise_timestep == pytest.approx(0.25)
_Pipe().record_denoise_step(5, normalized_timestep=0.7)
assert ctx.denoise_step_idx == 5 and ctx.denoise_timestep == pytest.approx(0.7)
@pytest.mark.parametrize(
"bk, field",
@@ -187,6 +191,7 @@ def test_mask_free_allowlist_gates_trtllm_default():
assert meta("WanPipeline").attention_mask_free is True
assert meta("WanVACEPipeline").attention_mask_free is True
assert meta("MiniMaxH3Pipeline").attention_mask_free is True
assert meta("FluxPipeline").attention_mask_free is False
assert meta(None).attention_mask_free is False
@@ -204,6 +209,91 @@ def test_masked_layer_raises():
_impl().forward_cuda(q, k, v, _Meta())
def test_packed_metadata_trims_padding_mask(monkeypatch):
b, s, h, d = 1, 8, 8, 128
q, k, v = (torch.zeros(b, s, h, d) for _ in range(3))
mask = torch.tensor([[True, True, True, True, True, True, False, False]])
cu_seqlens = torch.tensor([0, 6, 8], dtype=torch.int32)
metadata = AttentionMetadata(
attn_mask=mask,
extra={
"cu_seqlens_q": cu_seqlens,
"cu_seqlens_k": cu_seqlens,
"max_seqlen_q": 6,
"max_seqlen_k": 6,
},
)
captured = {}
def fake_attention(**kwargs):
captured.update(kwargs)
return torch.zeros_like(kwargs["query"])
monkeypatch.setattr(tg, "HAS_FLASHINFER", True)
monkeypatch.setattr(tg, "trtllm_ragged_attention_deepseek", fake_attention)
monkeypatch.setattr(TrtllmAttentionImpl, "_get_workspace", classmethod(lambda cls, device: torch.empty(0)))
out = _impl().forward_cuda(q, k, v, metadata)
assert out.shape == q.shape
assert captured["query"].shape[0] == 6
assert captured["key"].shape[0] == 6
assert captured["value"].shape[0] == 6
assert captured["batch_size"] == 1
assert captured["seq_lens"].tolist() == [6]
assert captured["cum_seq_lens_q"].tolist() == [0, 6]
assert captured["cum_seq_lens_kv"].tolist() == [0, 6]
assert captured["max_q_len"] == 6
assert captured["max_kv_len"] == 6
assert torch.count_nonzero(out[:, 6:]) == 0
def test_packed_metadata_rejects_non_prefix_mask(monkeypatch):
b, s, h, d = 1, 8, 8, 128
q, k, v = (torch.zeros(b, s, h, d) for _ in range(3))
cu_seqlens = torch.tensor([0, 6, 8], dtype=torch.int32)
metadata = AttentionMetadata(
attn_mask=torch.tensor([[True, True, False, True, True, True, False, False]]),
extra={
"cu_seqlens_q": cu_seqlens,
"cu_seqlens_k": cu_seqlens,
"max_seqlen_q": 6,
"max_seqlen_k": 6,
},
)
monkeypatch.setattr(tg, "HAS_FLASHINFER", True)
with pytest.raises(ValueError, match="prefix-valid"):
_impl().forward_cuda(q, k, v, metadata)
def test_packed_metadata_must_be_complete():
b, s, h, d = 1, 8, 8, 128
q, k, v = (torch.zeros(b, s, h, d) for _ in range(3))
metadata = AttentionMetadata(extra={"cu_seqlens_q": torch.tensor([0, s], dtype=torch.int32)})
with pytest.raises(ValueError, match="Incomplete packed TRTLLM attention metadata"):
_impl().forward_cuda(q, k, v, metadata)
def test_packed_metadata_must_cover_inputs(monkeypatch):
b, s, h, d = 1, 8, 8, 128
q, k, v = (torch.zeros(b, s, h, d) for _ in range(3))
cu_seqlens = torch.tensor([0, 6], dtype=torch.int32)
metadata = AttentionMetadata(
extra={
"cu_seqlens_q": cu_seqlens,
"cu_seqlens_k": cu_seqlens,
"max_seqlen_q": 6,
"max_seqlen_k": 6,
}
)
monkeypatch.setattr(tg, "HAS_FLASHINFER", True)
with pytest.raises(ValueError, match="must cover all Q/K/V tokens"):
_impl().forward_cuda(q, k, v, metadata)
def test_propagate_calibration_045_schema(monkeypatch):
import types
@@ -295,6 +385,30 @@ def test_bf16_dense_matches_sdpa():
assert rel < 0.01, f"BF16 dense rel err {rel:.4f} too high"
@requires_trtllm_attn
def test_bf16_packed_padding_matches_sdpa():
torch.manual_seed(0)
b, s, used, h, d = 1, 256, 192, 8, 128
scale = 1.0 / math.sqrt(d)
q, k, v = (torch.randn(b, s, h, d, device="cuda", dtype=torch.bfloat16) for _ in range(3))
cu_seqlens = torch.tensor([0, used, s], dtype=torch.int32, device="cuda")
metadata = AttentionMetadata(
attn_mask=torch.arange(s, device="cuda")[None] < used,
extra={
"cu_seqlens_q": cu_seqlens,
"cu_seqlens_k": cu_seqlens,
"max_seqlen_q": used,
"max_seqlen_k": used,
},
)
out = _impl().forward_cuda(q, k, v, metadata).float()
ref_used = _sdpa_ref(q[:, :used], k[:, :used], v[:, :used], scale)
used_rel = (out[:, :used] - ref_used).abs().mean() / ref_used.abs().mean()
assert used_rel < 0.01, f"BF16 packed valid-token rel err {used_rel:.4f} too high"
assert torch.count_nonzero(out[:, used:]) == 0
requires_sage = pytest.mark.skipif(
not (_has_trtllm_attn() and tg._sage_quantize_fn() is not None),
reason="requires Blackwell SM100+ GPU with flashinfer >= 0.6.16rc1 (trtllm_sage_attention_quantize)",
@@ -321,6 +435,47 @@ def test_sage_quant_matches_sdpa(dtype_qk):
assert rel < 0.1, f"SAGE({dtype_qk}) rel err {rel:.4f} too high"
@requires_sage
def test_sage_packed_non_aligned_length_matches_sdpa():
torch.manual_seed(0)
b, s, used, h, d = 1, 208, 194, 8, 128
scale = 1.0 / math.sqrt(d)
q, k, v = (torch.randn(b, s, h, d, device="cuda", dtype=torch.bfloat16) for _ in range(3))
cu_seqlens = torch.tensor([0, used, s], dtype=torch.int32, device="cuda")
metadata = AttentionMetadata(
attn_mask=torch.arange(s, device="cuda")[None] < used,
extra={
"cu_seqlens_q": cu_seqlens,
"cu_seqlens_k": cu_seqlens,
"max_seqlen_q": used,
"max_seqlen_k": used,
},
)
out = _impl(quant={"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16}).forward_cuda(q, k, v, metadata)
assert torch.isfinite(out).all()
ref = _sdpa_ref(q[:, :used], k[:, :used], v[:, :used], scale)
rel = (out[:, :used].float() - ref).abs().mean() / ref.abs().mean()
assert rel < 0.2, f"SAGE packed valid-token rel err {rel:.4f} too high"
assert torch.count_nonzero(out[:, used:]) == 0
@requires_sage
def test_sage_short_sequence_uses_dense_kernel():
torch.manual_seed(0)
b, s, h, d = 1, 14, 8, 128
scale = 1.0 / math.sqrt(d)
q, k, v = (torch.randn(b, s, h, d, device="cuda", dtype=torch.bfloat16) for _ in range(3))
impl = _impl(quant={"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16})
impl._sage_quantize_fn = lambda *args, **kwargs: pytest.fail("short sequence must not use SAGE")
out = impl.forward_cuda(q, k, v)
ref = _sdpa_ref(q, k, v, scale)
rel = (out.float() - ref).abs().mean() / ref.abs().mean()
assert torch.isfinite(out).all()
assert rel < 0.01, f"SAGE short-sequence dense fallback rel err {rel:.4f} too high"
@requires_trtllm_attn
def test_skip_tiny_threshold_matches_dense():
torch.manual_seed(0)
+31 -5
View File
@@ -86,9 +86,35 @@ class TestCacheDiTBackend:
# Verify cache-dit was enabled
assert backend.enabled is True
assert backend._refresh_func is not None
assert backend._refresh_funcs
mock_cache_dit.enable_cache.assert_called_once()
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
def test_enable_and_refresh_every_declared_dit(self, mock_cache_dit, mock_block_adapter):
"""Combined pipelines enable and refresh Cache-DiT for every DiT."""
pipeline = Mock()
pipeline.__class__.__name__ = "MiniMaxH3Pipeline"
pipeline._dit_modules = ["transformer", "transformers_ref"]
pipeline.transformer = Mock()
pipeline.transformers_ref = Mock()
for transformer in (pipeline.transformer, pipeline.transformers_ref):
transformer._cache_dit_adapter_config = CacheDiTAdapterConfig(
block_forward_patterns={"blocks": ForwardPattern.Pattern_3}
)
backend = CacheDiTBackend({"Fn_compute_blocks": 2})
backend.enable(pipeline)
backend.refresh(pipeline, num_inference_steps=20)
assert len(backend._refresh_funcs) == 2
assert mock_cache_dit.enable_cache.call_count == 2
assert mock_cache_dit.refresh_context.call_count == 2
assert {call.args[0] for call in mock_cache_dit.refresh_context.call_args_list} == {
pipeline.transformer,
pipeline.transformers_ref,
}
@patch("vllm_omni.diffusion.cache.cachedit.backend.logger")
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
def test_enable_without_model_adapter_uses_cache_dit_registry(self, mock_cache_dit, mock_logger):
@@ -106,7 +132,7 @@ class TestCacheDiTBackend:
backend.enable(pipeline)
assert backend.enabled is True
assert backend._refresh_func is not None
assert backend._refresh_funcs
assert mock_cache_dit.enable_cache.call_args.args[0] is pipeline.transformer
mock_logger.info.assert_any_call(
"Transformer %s does not declare _cache_dit_adapter_config; "
@@ -138,7 +164,7 @@ class TestCacheDiTBackend:
assert isinstance(exc_info.value.__cause__, ValueError)
assert backend.enabled is False
assert backend._refresh_func is None
assert backend._refresh_funcs == []
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
@@ -245,7 +271,7 @@ class TestCacheDiTBackend:
backend.enable(mock_pipeline)
assert backend.enabled is True
assert backend._refresh_func is not None
assert backend._refresh_funcs
mock_block_adapter.assert_called_once()
adapter_kwargs = mock_block_adapter.call_args.kwargs
assert adapter_kwargs["transformer"] is mock_pipeline.model
@@ -285,7 +311,7 @@ class TestCacheDiTBackend:
backend.enable(mock_pipeline)
assert backend.enabled is True
assert backend._refresh_func is not None
assert backend._refresh_funcs
mock_block_adapter.assert_called_once()
adapter_kwargs = mock_block_adapter.call_args.kwargs
assert adapter_kwargs["transformer"] is mock_pipeline.transformer
+41
View File
@@ -2,6 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for LayerNorm and RMSNorm custom ops in diffusion layers."""
import sys
import types
import pytest
import torch
from pytest_mock import MockerFixture
@@ -299,6 +302,44 @@ def test_rmsnorm_numerical_correctness():
torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5)
def test_rmsnorm_dtype_matches_parameter_and_fp32_accumulation_contract():
"""dtype controls gamma storage; reduced-precision inputs still use fp32 accumulation."""
from vllm_omni.diffusion.layers.norm import RMSNorm
hidden_size = 64
eps = 1e-6
norm = RMSNorm(hidden_size, eps=eps, dtype=torch.bfloat16)
x = torch.randn(2, 4, hidden_size, dtype=torch.bfloat16)
assert norm.weight.dtype == torch.bfloat16
x_fp32 = x.float()
expected = (x_fp32 * torch.rsqrt(x_fp32.square().mean(-1, keepdim=True) + eps) * norm.weight.float()).to(x.dtype)
torch.testing.assert_close(norm.forward_native(x), expected, atol=0, rtol=0)
def test_rmsnorm_npu_receives_gamma_with_configured_dtype(monkeypatch: pytest.MonkeyPatch):
"""The NPU fused op receives the bf16 checkpoint gamma used by H3."""
from vllm_omni.diffusion.layers.norm import RMSNorm
captured: dict[str, torch.Tensor] = {}
def npu_rms_norm(x: torch.Tensor, gamma: torch.Tensor, epsilon: float):
captured["gamma"] = gamma
captured["epsilon"] = torch.tensor(epsilon)
return (x,)
monkeypatch.setitem(sys.modules, "torch_npu", types.SimpleNamespace(npu_rms_norm=npu_rms_norm))
norm = RMSNorm(64, eps=1e-5, dtype=torch.bfloat16)
x = torch.randn(2, 4, 64, dtype=torch.bfloat16)
assert norm.forward_npu(x) is x
assert captured["gamma"] is norm.weight
assert captured["gamma"].dtype == torch.bfloat16
assert captured["epsilon"].item() == pytest.approx(1e-5)
# ── RMSNorm compile-path regression tests ──
+34 -1
View File
@@ -10,10 +10,13 @@ forward_hip, and apply_rotary_emb_mindiesd paths.
from __future__ import annotations
import sys
import types
import pytest
import torch
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
from vllm_omni.diffusion.layers.rope import RotaryEmbedding, apply_rotary_emb_mindiesd
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -234,3 +237,33 @@ class TestRotaryEmbeddingNativeShapeRegression:
output = rope.forward_native(x, cos, sin)
assert output.shape == x.shape
assert not torch.isnan(output).any()
def test_mindie_full_dim_rope_keeps_h3_cos_sin_layout(monkeypatch: pytest.MonkeyPatch) -> None:
"""H3 passes complete rotary dims to MindIE instead of the CUDA half-dim layout."""
captured: dict[str, object] = {}
def rotary_position_embedding(x, cos, sin, **kwargs):
captured.update(x=x, cos=cos, sin=sin, kwargs=kwargs)
return x
monkeypatch.setitem(
sys.modules,
"mindiesd",
types.SimpleNamespace(rotary_position_embedding=rotary_position_embedding),
)
x = torch.randn(11, 3, 96, dtype=torch.bfloat16)
cos = torch.randn(11, 96, dtype=torch.bfloat16)
sin = torch.randn(11, 96, dtype=torch.bfloat16)
actual = apply_rotary_emb_mindiesd(x, cos, sin, interleaved=False, half_head_dim=False)
assert actual is x
assert captured["x"] is x
assert captured["cos"] is cos
assert captured["sin"] is sin
assert captured["kwargs"] == {
"rotated_mode": "rotated_half",
"head_first": False,
"fused": True,
}
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from multiprocessing.reduction import ForkingPickler
from types import SimpleNamespace
from unittest.mock import Mock
@@ -24,12 +25,343 @@ def test_pipeline_import_registry_and_component_discovery():
"pipeline_minimax_h3",
"MiniMaxH3Pipeline",
)
assert _DIFFUSION_MODELS["MiniMaxH3ModularPipeline"] == _DIFFUSION_MODELS["MiniMaxH3Pipeline"]
assert _DIFFUSION_POST_PROCESS_FUNCS["MiniMaxH3Pipeline"] == "get_minimax_h3_post_process_func"
assert MiniMaxH3Pipeline._dit_modules == ["transformer"]
assert (
_DIFFUSION_POST_PROCESS_FUNCS["MiniMaxH3ModularPipeline"] == _DIFFUSION_POST_PROCESS_FUNCS["MiniMaxH3Pipeline"]
)
assert MiniMaxH3Pipeline._dit_modules == ["transformer", "transformers_ref"]
assert MiniMaxH3Pipeline._encoder_modules == ["text_encoder"]
assert MiniMaxH3Pipeline._vae_modules == ["video_vae", "audio_vae"]
def _write_partition_index(path, *, partition, tasks):
path.mkdir(parents=True)
(path / "model_index.json").write_text(
json.dumps(
{
"_class_name": "MiniMaxH3Pipeline",
"_diffusers_version": "0.32.2",
"_minimax_h3": {
"partition": partition,
"tasks": tasks,
"sigma_shift_scales": {"video": 12.0, "audio": 3.0},
},
}
),
encoding="utf-8",
)
def test_modular_diffusers_index_is_resolved_generically(tmp_path):
from vllm_omni.diffusion.data import OmniDiffusionConfig, resolve_model_class_name
from vllm_omni.diffusion.utils.hf_utils import is_diffusion_model
from vllm_omni.entrypoints.utils import resolve_model_config_path
(tmp_path / "modular_model_index.json").write_text(
json.dumps(
{
"_class_name": "MiniMaxH3ModularPipeline",
"_diffusers_version": "0.36.0.dev0",
}
),
encoding="utf-8",
)
assert is_diffusion_model(str(tmp_path))
assert resolve_model_class_name(str(tmp_path)) == "MiniMaxH3ModularPipeline"
assert resolve_model_config_path(str(tmp_path)) is None
config = OmniDiffusionConfig(model=str(tmp_path))
config.enrich_config()
assert config.model_class_name == "MiniMaxH3ModularPipeline"
assert config.supports_multimodal_inputs
assert config.max_multimodal_image_inputs == 9
assert config.supports_mixed_reference_inputs
@pytest.mark.parametrize(
("task_type", "partition"),
[
(None, "combined"),
("auto", "combined"),
("t2va", "fl2va"),
("fl2va", "fl2va"),
("ref2va", "ref2va"),
],
)
def test_startup_task_selects_weight_partition(task_type, partition):
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import _minimax_h3_partition_for_task
assert _minimax_h3_partition_for_task(task_type) == partition
def test_startup_auto_task_uses_explicit_local_partition(tmp_path):
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import _minimax_h3_partition_for_task
for directory, partition, tasks in (
("FL2VA", "fl2va", ["t2va", "fl2va"]),
("Ref2VA", "ref2va", ["ref2va"]),
):
model_path = tmp_path / directory
_write_partition_index(model_path, partition=partition, tasks=tasks)
assert _minimax_h3_partition_for_task(None, str(model_path)) == partition
assert _minimax_h3_partition_for_task("auto", str(model_path)) == partition
assert _minimax_h3_partition_for_task("combined", str(model_path)) == "combined"
def test_startup_task_rejects_unsupported_value():
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import _minimax_h3_partition_for_task
with pytest.raises(ValueError, match="task_type must be one of"):
_minimax_h3_partition_for_task("unsupported")
def test_combined_task_inference_and_transformer_routing():
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.partition = "combined"
pipeline.supported_tasks = frozenset({"t2va", "fl2va", "ref2va"})
pipeline.transformer = torch.nn.Identity()
pipeline.transformers_ref = torch.nn.Linear(1, 1)
assert pipeline._resolve_task(None, {}) == "t2va"
assert pipeline._resolve_task(None, {"image": object()}) == "fl2va"
assert pipeline._resolve_task(None, {"audio": object()}) == "ref2va"
assert pipeline._resolve_task(None, {"video": object()}) == "ref2va"
pipeline.partition = "ref2va"
pipeline.supported_tasks = frozenset({"ref2va"})
assert pipeline._resolve_task(None, {"image": object()}) == "ref2va"
pipeline.partition = "combined"
pipeline.supported_tasks = frozenset({"t2va", "fl2va", "ref2va"})
assert pipeline._transformer_for_task("fl2va") is pipeline.transformer
assert pipeline._transformer_for_task("ref2va") is pipeline.transformers_ref
@pytest.mark.parametrize(
(
"task_type",
"expected_partition",
"expected_dits",
"component_partition",
"source_partitions",
"expected_tasks",
),
[
(None, "combined", 2, "FL2VA", ["FL2VA", "Ref2VA"], {"t2va", "fl2va", "ref2va"}),
("fl2va", "fl2va", 1, "FL2VA", ["FL2VA"], {"t2va", "fl2va"}),
("ref2va", "ref2va", 1, "Ref2VA", ["Ref2VA"], {"ref2va"}),
],
)
def test_pipeline_loads_task_selected_dits_and_shared_components_once(
monkeypatch,
tmp_path,
task_type,
expected_partition,
expected_dits,
component_partition,
source_partitions,
expected_tasks,
):
from vllm_omni.diffusion.data import (
DiffusionParallelConfig,
OmniDiffusionConfig,
)
from vllm_omni.diffusion.models.minimax_h3 import (
pipeline_minimax_h3 as pipeline_module,
)
_write_partition_index(
tmp_path / "FL2VA",
partition="fl2va",
tasks=["t2va", "fl2va"],
)
_write_partition_index(
tmp_path / "Ref2VA",
partition="ref2va",
tasks=["ref2va"],
)
for partition_name in ("FL2VA", "Ref2VA"):
for component in (
"transformer",
"tokenizer",
"processor",
"text_encoder",
"video_vae",
"audio_vae",
):
(tmp_path / partition_name / component).mkdir()
created = {"dit": [], "text_encoder": [], "video_vae": [], "audio_vae": []}
class FakeModule(torch.nn.Module):
def __init__(self, *args, **kwargs):
super().__init__()
class FakeDiT(FakeModule):
def __init__(self, *args, **kwargs):
super().__init__()
created["dit"].append((args, kwargs))
def component_factory(name):
def create(path, *args, **kwargs):
created[name].append(str(path))
return FakeModule()
return create
tokenizer_calls = []
processor_calls = []
tokenizer_cls = Mock(spec=pipeline_module.Qwen2TokenizerFast)
tokenizer_cls.from_pretrained.side_effect = lambda *args, **kwargs: (
tokenizer_calls.append((args, kwargs)) or object()
)
processor_cls = Mock(spec=pipeline_module.Qwen3VLProcessor)
processor_cls.from_pretrained.side_effect = lambda *args, **kwargs: (
processor_calls.append((args, kwargs)) or object()
)
monkeypatch.setattr(pipeline_module, "MiniMaxH3DiTModel", FakeDiT)
monkeypatch.setattr(
pipeline_module,
"MiniMaxH3Qwen3VLEncoder",
component_factory("text_encoder"),
)
monkeypatch.setattr(
pipeline_module,
"MiniMaxH3VideoVAE",
component_factory("video_vae"),
)
monkeypatch.setattr(
pipeline_module,
"MiniMaxH3AudioVAE",
component_factory("audio_vae"),
)
monkeypatch.setattr(
pipeline_module,
"Qwen2TokenizerFast",
tokenizer_cls,
)
monkeypatch.setattr(
pipeline_module,
"Qwen3VLProcessor",
processor_cls,
)
monkeypatch.setattr(
pipeline_module,
"_dit_rank_world",
lambda: (None, 0, 1),
)
monkeypatch.setattr(
pipeline_module,
"get_local_device",
lambda: torch.device("cpu"),
)
download_calls = []
def fake_download(**kwargs):
download_calls.append(kwargs)
return str(tmp_path)
monkeypatch.setattr(
pipeline_module,
"download_weights_from_hf_specific",
fake_download,
)
od_config = OmniDiffusionConfig(
model="MiniMaxAI/MiniMax-H3",
revision=None,
task_type=task_type,
quantization_config=None,
parallel_config=DiffusionParallelConfig(
cfg_parallel_size=1,
text_encoder_tp_size=1,
),
)
pipeline = pipeline_module.MiniMaxH3Pipeline(od_config=od_config)
assert pipeline.partition == expected_partition
assert pipeline.supported_tasks == expected_tasks
assert len(created["dit"]) == expected_dits
component_path = tmp_path / component_partition
assert created["text_encoder"] == [str(component_path / "text_encoder")]
assert created["video_vae"] == [str(component_path / "video_vae")]
assert created["audio_vae"] == [str(component_path / "audio_vae")]
assert len(tokenizer_calls) == 1
assert len(processor_calls) == 1
expected_dit_modules = ["transformer", "transformers_ref"] if expected_dits == 2 else ["transformer"]
assert pipeline._dit_modules == expected_dit_modules
assert hasattr(pipeline, "transformers_ref") is (expected_dits == 2)
if expected_partition == "ref2va":
assert pipeline._transformer_for_task("ref2va") is pipeline.transformer
assert [source.model_or_path for source in pipeline.weights_sources] == [
str(tmp_path / partition_name) for partition_name in source_partitions
]
expected_patterns = (
pipeline_module.MINIMAX_H3_DOWNLOAD_PATTERNS
if expected_partition == "combined"
else pipeline_module.MINIMAX_H3_TASK_DOWNLOAD_PATTERNS[expected_partition]
)
assert download_calls == [
{
"model_name_or_path": "MiniMaxAI/MiniMax-H3",
"cache_dir": None,
"allow_patterns": expected_patterns,
"revision": None,
"require_all": True,
}
]
def test_combined_weight_loader_routes_each_contiguous_partition():
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
class FakeTransformer(torch.nn.Module):
def __init__(self):
super().__init__()
self.loaded = []
self.post_load_calls = 0
def load_weights(self, weights):
self.loaded = [name for name, _ in weights]
return set(self.loaded)
def post_load_weights(self):
self.post_load_calls += 1
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.transformer = FakeTransformer()
pipeline.transformers_ref = FakeTransformer()
pipeline.text_encoder = torch.nn.Identity()
pipeline.video_vae = torch.nn.Identity()
pipeline.audio_vae = torch.nn.Identity()
loaded = pipeline.load_weights(
iter(
[
("transformer.a", torch.ones(1)),
("transformer.b", torch.ones(1)),
("transformers_ref.a", torch.ones(1)),
]
)
)
assert pipeline.transformer.loaded == ["a", "b"]
assert pipeline.transformers_ref.loaded == ["a"]
assert pipeline.transformer.post_load_calls == 1
assert pipeline.transformers_ref.post_load_calls == 1
assert loaded == {"transformer.a", "transformer.b", "transformers_ref.a"}
def test_dlo_offload_plan_includes_token_refiner():
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
assert MiniMaxH3Pipeline._offload_plan.offload_submodules == {"token_refiner": "blocks"}
assert MiniMaxH3Pipeline._offload_plan.resident_dit_paths == frozenset({"transformer"})
def test_joint_postprocess_is_multiprocessing_picklable():
from vllm_omni.diffusion.models.minimax_h3 import (
get_minimax_h3_post_process_func,
@@ -87,6 +419,87 @@ def test_shifted_sigma_schedule_matches_reference_values():
)
def test_cudnn_packed_attention_uses_python_length_without_padding_mask():
from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import (
MiniMaxH3Attention,
)
class FakeBackend:
supports_prefix_kv_slicing = True
class FakeAttention(torch.nn.Module):
attn_backend = FakeBackend
def __init__(self):
super().__init__()
self.metadata = None
def forward(self, query, key, value, metadata):
self.metadata = metadata
return query
attention = object.__new__(MiniMaxH3Attention)
torch.nn.Module.__init__(attention)
attention.attention = FakeAttention()
# Model-side Ulysses shards rows before Attention gathers them again.
# The Python packed_total must therefore remain global even though q is
# local here.
q = torch.randn(4, 2, 4)
cu_seqlens = torch.tensor([0, 5, 8], dtype=torch.int32)
output = attention._run_packed_attention(
q,
q,
q,
cu_seqlens=cu_seqlens,
max_seqlen=5,
packed_total=8,
)
assert output.shape == q.shape
assert attention.attention.metadata.attn_mask is None
assert attention.attention.metadata.extra["valid_kv_length"] == 5
def test_packed_attention_keeps_padding_mask_for_other_backends():
from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import (
MiniMaxH3Attention,
)
class FakeBackend:
supports_prefix_kv_slicing = False
class FakeAttention(torch.nn.Module):
attn_backend = FakeBackend
def __init__(self):
super().__init__()
self.metadata = None
def forward(self, query, key, value, metadata):
self.metadata = metadata
return query
attention = object.__new__(MiniMaxH3Attention)
torch.nn.Module.__init__(attention)
attention.attention = FakeAttention()
q = torch.randn(8, 2, 4)
attention._run_packed_attention(
q,
q,
q,
cu_seqlens=torch.tensor([0, 5, 8], dtype=torch.int32),
max_seqlen=5,
packed_total=8,
)
assert torch.equal(
attention.attention.metadata.attn_mask,
torch.tensor([[True, True, True, True, True, False, False, False]]),
)
def test_reference_image_resize_contract():
from PIL import Image
@@ -293,6 +706,142 @@ def test_layerwise_offload_releases_text_encoder():
pipeline.text_encoder.offload_to_cpu.assert_called_once_with()
def test_distributed_layerwise_offload_releases_text_encoder():
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.od_config = SimpleNamespace(
enable_cpu_offload=False,
enable_layerwise_offload=False,
enable_distributed_layerwise_offload=True,
)
pipeline.text_encoder = Mock()
expected = torch.ones(2, 3)
pipeline.text_encoder.encode_ids.return_value = expected
actual = pipeline._encode_text_hidden(torch.tensor([1, 2]), {})
assert actual is expected
pipeline.text_encoder.load_to_device.assert_called_once_with()
pipeline.text_encoder.offload_to_cpu.assert_called_once_with()
def test_distributed_layerwise_offload_stages_vae_component():
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.od_config = SimpleNamespace(
enable_layerwise_offload=False,
enable_distributed_layerwise_offload=True,
)
component = Mock()
with pipeline._component_on_device(component):
component.load_to_device.assert_called_once_with()
component.offload_to_cpu.assert_not_called()
component.offload_to_cpu.assert_called_once_with()
def test_distributed_layerwise_resident_blocks_are_stage_scoped():
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
controller = Mock()
pipeline._dlo_residency_controller = controller
with pipeline._resident_dit_layers_on_device():
controller.load_resident_layers.assert_called_once_with()
controller.offload_resident_layers.assert_not_called()
controller.offload_resident_layers.assert_called_once_with()
def test_distributed_layerwise_resident_blocks_release_on_failure():
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
controller = Mock()
pipeline._dlo_residency_controller = controller
with pytest.raises(RuntimeError, match="denoise failed"):
with pipeline._resident_dit_layers_on_device():
raise RuntimeError("denoise failed")
controller.load_resident_layers.assert_called_once_with()
controller.offload_resident_layers.assert_called_once_with()
def test_distributed_layerwise_resident_blocks_can_be_skipped():
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
controller = Mock()
pipeline._dlo_residency_controller = controller
with pipeline._resident_dit_layers_on_device(enabled=False):
pass
controller.load_resident_layers.assert_not_called()
controller.offload_resident_layers.assert_not_called()
def test_encoder_layerwise_offload_keeps_tp_blocks_rank_local(monkeypatch):
from vllm_omni.diffusion.models.minimax_h3.encoder import (
MiniMaxH3Qwen3VLEncoder,
)
class Stack(torch.nn.Module):
def __init__(self, count):
super().__init__()
self.blocks = torch.nn.ModuleList([torch.nn.Linear(2, 2) for _ in range(count)])
class TextStack(torch.nn.Module):
def __init__(self, count):
super().__init__()
self.layers = torch.nn.ModuleList([torch.nn.Linear(2, 2) for _ in range(count)])
hooks = []
def fake_apply(block, next_block, device, stream, pin_memory):
hook = SimpleNamespace(
block=block,
next_block=next_block,
device=device,
stream=stream,
pin_memory=pin_memory,
_prev_hook=None,
offload_layer=Mock(),
)
hooks.append(hook)
return hook
monkeypatch.setattr(
"vllm_omni.diffusion.offloader.layerwise_backend.apply_block_hook",
fake_apply,
)
monkeypatch.setattr(
"vllm_omni.platforms.current_omni_platform.Stream",
Mock(return_value="copy-stream"),
)
encoder = object.__new__(MiniMaxH3Qwen3VLEncoder)
torch.nn.Module.__init__(encoder)
encoder.device_target = torch.device("cpu")
encoder.vision = Stack(2)
encoder.text_model = TextStack(3)
encoder.enable_omni_layerwise_offload(pin_memory=False)
assert len(hooks) == 5
assert all(hook._prev_hook is not None for hook in hooks)
assert all(hook.device == torch.device("cpu") for hook in hooks)
def test_video_vae_keeps_reference_fp32_weights(monkeypatch):
from vllm_omni.diffusion.models.minimax_h3 import vae as vae_module
@@ -324,6 +873,82 @@ def test_video_vae_keeps_reference_fp32_weights(monkeypatch):
assert next(video_vae.parameters()).dtype == torch.float32
def test_video_vae_can_load_on_cpu_for_staged_gpu_residency(monkeypatch):
from vllm_omni.diffusion.models.minimax_h3 import vae as vae_module
class FakeRemote(torch.nn.Module):
def __init__(self):
super().__init__()
self.model = torch.nn.Linear(1, 1)
monkeypatch.setattr(
vae_module,
"_load_component_config",
lambda _path: {
"latent_channels": 1,
"latents_mean": [0.0],
"latents_std": [1.0],
},
)
monkeypatch.setattr(
vae_module,
"_load_remote_component",
lambda _path, _config: FakeRemote(),
)
video_vae = vae_module.MiniMaxH3VideoVAE(
"unused",
device=torch.device("meta"),
load_device=torch.device("cpu"),
)
assert next(video_vae.parameters()).device.type == "cpu"
assert video_vae._device_target.type == "meta"
def test_video_vae_uses_snapshot_stager_for_accelerator_target(monkeypatch):
from vllm_omni.diffusion.models.minimax_h3 import vae as vae_module
class FakeRemote(torch.nn.Module):
def __init__(self):
super().__init__()
self.model = torch.nn.Linear(1, 1)
stager = Mock()
stager_cls = Mock(return_value=stager)
monkeypatch.setattr(
vae_module,
"_load_component_config",
lambda _path: {
"latent_channels": 1,
"latents_mean": [0.0],
"latents_std": [1.0],
},
)
monkeypatch.setattr(
vae_module,
"_load_remote_component",
lambda _path, _config: FakeRemote(),
)
monkeypatch.setattr(vae_module, "PinnedModuleStager", stager_cls)
video_vae = vae_module.MiniMaxH3VideoVAE(
"unused",
device=torch.device("cuda"),
load_device=torch.device("cpu"),
)
video_vae.load_to_device()
video_vae.offload_to_cpu()
stager_cls.assert_called_once_with(
video_vae.remote,
torch.device("cuda"),
pin_memory=True,
)
stager.load.assert_called_once_with()
stager.offload.assert_called_once_with()
def test_video_vae_encode_uses_configured_parallel_tiling():
from vllm_omni.diffusion.models.minimax_h3.vae import (
MiniMaxH3VideoVAE,
@@ -61,6 +61,33 @@ def test_packed_attention_is_a_regional_compile_boundary():
assert getattr(MiniMaxH3Attention._run_packed_attention, "_torchdynamo_disable", False)
def test_h3_fused_rope_matches_reference_and_preserves_unrotated_dims():
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import (
MiniMaxH3Attention,
)
attention = object.__new__(MiniMaxH3Attention)
nn.Module.__init__(attention)
attention.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)
attention.rope._forward_method = attention.rope.forward_native
x = torch.randn(11, 3, 128, dtype=torch.bfloat16)
freqs_half = torch.randn(11, 48)
freqs = torch.cat((freqs_half, freqs_half), dim=-1)
actual = attention._apply_rope(x, freqs)
cos = torch.cos(freqs).to(x.dtype).unsqueeze(1)
sin = torch.sin(freqs).to(x.dtype).unsqueeze(1)
x_rot = x[..., :96]
x1, x2 = x_rot.chunk(2, dim=-1)
expected_rot = x_rot * cos + torch.cat((-x2, x1), dim=-1) * sin
expected = torch.cat((expected_rot, x[..., 96:]), dim=-1)
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
torch.testing.assert_close(actual[..., 96:], x[..., 96:], atol=0, rtol=0)
@pytest.mark.parametrize(
("tp_size", "message"),
[
@@ -135,6 +135,7 @@ def _generate_joint_output(omni, config: QualityTestConfig):
num_frames=config.num_frames,
guidance_scale=config.guidance_scale,
sigmas=config.sigmas,
extra_args={"aspect_ratio": "16:9"},
),
)
main_process_peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
@@ -27,7 +27,9 @@ from vllm_omni.diffusion.offloader.block_discovery import (
from vllm_omni.diffusion.offloader.distributed_layerwise_backend import (
DistributedLayerwiseOffloadBackend,
DistributedLayerwiseOffloadHook,
PinnedResidentLayerGroup,
)
from vllm_omni.diffusion.offloader.module_residency import PinnedModuleStager
from vllm_omni.diffusion.offloader.offload_plan import OffloadPlan, get_offload_plan
from vllm_omni.platforms import current_omni_platform
@@ -97,6 +99,7 @@ def patched_offload_runtime(monkeypatch):
monkeypatch.setattr(dist_backend_module.current_omni_platform, "Event", DummyEvent)
monkeypatch.setattr(dist_backend_module.current_omni_platform, "current_stream", lambda: DummyStream())
monkeypatch.setattr(dist_backend_module.current_omni_platform, "stream", dummy_stream)
monkeypatch.setattr(dist_backend_module.current_omni_platform, "synchronize", lambda: None)
class TinyBlock(nn.Module):
@@ -308,6 +311,164 @@ class TestDistributedLayerwiseOffloadHook:
assert shard1.numel() == 2
assert torch.equal(shard1, torch.tensor([102.0, 0.0]))
def test_mmap_layout_transform_is_applied_before_sharding(self, dist_group, patched_offload_runtime):
block = nn.Module()
block.weight = nn.Parameter(torch.zeros(4, dtype=torch.float32))
next_block = nn.Module()
next_block.weight = nn.Parameter(torch.arange(4, dtype=torch.float32))
next_block.weight.mmap_weight_transform = lambda tensor: tensor.flip(0)
next_block.weight.mmap_weight_transform_pending = True
hook = DistributedLayerwiseOffloadHook(
next_block=next_block,
device=torch.device("cpu"),
dp_group=None,
dp_size=2,
rank=0,
pin_memory=False,
)
hook.initialize_hook(block)
assert torch.equal(hook.cpu_shards[torch.float32], torch.tensor([3.0, 2.0]))
def test_regular_loader_weight_does_not_reapply_mmap_transform(self, dist_group, patched_offload_runtime):
block = nn.Module()
block.weight = nn.Parameter(torch.zeros(4, dtype=torch.float32))
next_block = nn.Module()
next_block.weight = nn.Parameter(torch.arange(4, dtype=torch.float32))
next_block.weight.mmap_weight_transform = lambda tensor: tensor.flip(0)
hook = DistributedLayerwiseOffloadHook(
next_block=next_block,
device=torch.device("cpu"),
dp_group=None,
dp_size=2,
rank=0,
pin_memory=False,
)
hook.initialize_hook(block)
assert torch.equal(hook.cpu_shards[torch.float32], torch.tensor([0.0, 1.0]))
class TestPinnedResidentLayerGroup:
def test_load_offload_reuses_pinned_master_weights(self, patched_offload_runtime):
blocks = [nn.Linear(2, 2), nn.Linear(2, 2)]
expected = []
for index, block in enumerate(blocks):
block.weight.data.fill_(index + 1)
block.bias.data.fill_(index + 3)
expected.append((block.weight.detach().clone(), block.bias.detach().clone()))
group = PinnedResidentLayerGroup(
blocks,
device=torch.device("cpu"),
copy_stream=DummyStream(),
pin_memory=False,
)
assert all(block.weight.numel() == 0 for block in blocks)
group.load()
for block, (weight, bias) in zip(blocks, expected):
assert torch.equal(block.weight, weight)
assert torch.equal(block.bias, bias)
# Device-side changes must not overwrite the persistent host master.
blocks[0].weight.data.zero_()
group.offload()
assert all(block.weight.numel() == 0 for block in blocks)
group.load()
for block, (weight, bias) in zip(blocks, expected):
assert torch.equal(block.weight, weight)
assert torch.equal(block.bias, bias)
group.offload()
def test_mmap_loader_attrs_survive_to_empty_parameter_replacement(self):
module = nn.Linear(2, 2, bias=False)
def transform(tensor):
return tensor.flip(0)
module.weight.mmap_weight_transform = transform
backend = object.__new__(DistributedLayerwiseOffloadBackend)
backend._remember_mmap_param_attrs(module)
module.to_empty(device="meta")
replacement = nn.Parameter(torch.arange(4).reshape(2, 2).float())
backend._attach_mmap_param_attrs("weight", replacement, module.weight)
assert replacement.mmap_weight_transform is transform
assert replacement.mmap_weight_transform_pending is True
def test_shared_allgather_output_is_narrowed_to_current_block(
self, dist_group, patched_offload_runtime, monkeypatch
):
block = nn.Module()
block.weight = nn.Parameter(torch.zeros(3, dtype=torch.float32))
next_block = nn.Module()
next_block.weight = nn.Parameter(torch.arange(3, dtype=torch.float32))
hook = DistributedLayerwiseOffloadHook(
next_block=next_block,
device=torch.device("cpu"),
dp_group=object(),
dp_size=2,
rank=0,
pin_memory=False,
)
hook.initialize_hook(block)
# Simulate unified buffers sized for a larger heterogeneous block.
hook.gpu_buffers = [
{torch.float32: torch.empty(12)},
{torch.float32: torch.empty(12)},
]
hook.gpu_shard_buffers = [
{torch.float32: torch.empty(6)},
{torch.float32: torch.empty(6)},
]
output_sizes = []
def fake_allgather(output, local_shard, *, group):
del group
output_sizes.append(output.numel())
output.zero_()
output[: local_shard.numel()].copy_(local_shard)
monkeypatch.setattr(torch.distributed, "all_gather_into_tensor", fake_allgather)
hook.prefetch_layer(slot=0, non_blocking=False)
assert output_sizes == [4]
class TestPinnedModuleStager:
def test_offload_rebinds_immutable_cpu_master(self, patched_offload_runtime):
module = nn.Linear(2, 2)
expected_weight = module.weight.detach().clone()
expected_bias = module.bias.detach().clone()
stager = PinnedModuleStager(
module,
torch.device("cpu"),
pin_memory=False,
copy_stream=DummyStream(),
)
stager.load()
module.weight.data.zero_()
module.bias.data.zero_()
stager.offload()
assert torch.equal(module.weight, expected_weight)
assert torch.equal(module.bias, expected_bias)
stager.load()
assert torch.equal(module.weight, expected_weight)
assert torch.equal(module.bias, expected_bias)
stager.offload()
class _DummyBlock(nn.Module):
def __init__(self):
@@ -504,6 +665,81 @@ class TestCrossGroupSharedBuffer:
assert not sync_called[0], "Should skip sync-prefetch when slot_group matches (non-contaminated)"
class TestAllGatherOutputSlicing:
"""Regression test: with heterogeneous block sizes (e.g. MiniMax H3's
token_refiner with blocks of ~1231MB and ~239MB), the shared AllGather
output buffers are sized to the *largest* block across all groups.
prefetch_layer must slice the output buffer down to each block's actual
AllGather output size, otherwise output.numel() != dp_size * input.numel()
and all_gather_into_tensor raises during enable()'s first prefetch.
"""
def test_prefetch_slices_shared_output_buffer_to_block_size(self, dist_group, patched_offload_runtime, monkeypatch):
dp_size = 2
# Heterogeneous block sizes: 8 vs 32 elements.
small_block = nn.Module()
small_block.weight = nn.Parameter(torch.arange(8, dtype=torch.float32))
large_block = nn.Module()
large_block.weight = nn.Parameter(torch.arange(100, 132, dtype=torch.float32))
def make_hook(next_block: nn.Module) -> DistributedLayerwiseOffloadHook:
hook = DistributedLayerwiseOffloadHook(
next_block=next_block,
device=torch.device("cpu"),
dp_group=dist.group.WORLD, # non-None -> AllGather path (mocked below)
dp_size=dp_size,
rank=0,
pin_memory=False,
)
current_block = nn.Module()
current_block.weight = nn.Parameter(torch.zeros(4, dtype=torch.float32))
hook.initialize_hook(current_block)
return hook
hook_small = make_hook(small_block)
hook_large = make_hook(large_block)
hooks = [hook_small, hook_large]
# Assign max-sized shared buffers exactly as backend.enable() does.
shared_buffers = DistributedLayerwiseOffloadBackend._allocate_shared_buffers(hooks)
shared_shard_buffers = DistributedLayerwiseOffloadBackend._allocate_shared_shard_buffers(hooks)
for hook in hooks:
hook.gpu_buffers = shared_buffers
hook.gpu_shard_buffers = shared_shard_buffers
# Shared output buffer must be sized to the largest block (32).
assert shared_buffers[0][torch.float32].numel() == 32
# Fake AllGather that enforces the real torch size contract and
# emulates reconstruction of the full block from all ranks' shards.
full_values = {
8: torch.arange(8, dtype=torch.float32),
32: torch.arange(100, 132, dtype=torch.float32),
}
ag_output_numels: list[int] = []
def fake_all_gather(output, input, group=None):
assert output.numel() == dp_size * input.numel(), (
f"AllGather contract violated: output.numel()={output.numel()} "
f"!= {dp_size} * input.numel()={input.numel()}"
)
ag_output_numels.append(output.numel())
output.copy_(full_values[output.numel()])
monkeypatch.setattr(dist, "all_gather_into_tensor", fake_all_gather)
# Small block: output must be sliced to dp_size * shard = 8, not 32.
hook_small.prefetch_layer(slot=0, non_blocking=False)
assert ag_output_numels[-1] == 8
assert torch.equal(small_block.weight, full_values[8])
# Large block: uses the full max-sized buffer.
hook_large.prefetch_layer(slot=1, non_blocking=False)
assert ag_output_numels[-1] == 32
assert torch.equal(large_block.weight, full_values[32])
class TestOffloadPlan:
"""Test declarative OffloadPlan metadata."""
@@ -517,6 +753,7 @@ class TestOffloadPlan:
plan = OffloadPlan(
block_attrs={"transformer": ("gen_layers",)},
offload_submodules={"context_encoder": "layers"},
resident_dit_paths=frozenset({"transformer"}),
)
class PipelineWithPlan(nn.Module):
@@ -527,12 +764,14 @@ class TestOffloadPlan:
assert result is plan
assert result.block_attrs == {"transformer": ("gen_layers",)}
assert result.offload_submodules == {"context_encoder": "layers"}
assert result.resident_dit_paths == frozenset({"transformer"})
def test_offload_plan_defaults_to_empty(self):
"""OffloadPlan with no arguments should have empty dicts."""
plan = OffloadPlan()
assert plan.block_attrs == {}
assert plan.offload_submodules == {}
assert plan.resident_dit_paths == frozenset()
def test_offload_plan_is_frozen(self):
"""OffloadPlan should be immutable (frozen=True)."""
@@ -540,6 +779,50 @@ class TestOffloadPlan:
with pytest.raises(Exception):
plan.block_attrs = {"x": ("y",)} # type: ignore
def test_all_resident_dit_still_prepares_planned_submodules(self, patched_offload_runtime):
class TokenRefiner(nn.Module):
def __init__(self):
super().__init__()
self.blocks = nn.ModuleList([nn.Linear(2, 2), nn.Linear(2, 2)])
class Transformer(nn.Module):
_layerwise_offload_blocks_attrs = ["blocks"]
def __init__(self):
super().__init__()
self.blocks = nn.ModuleList([nn.Linear(2, 2), nn.Linear(2, 2)])
self.token_refiner = TokenRefiner()
class Pipeline(nn.Module):
_offload_plan = OffloadPlan(
offload_submodules={"token_refiner": "blocks"},
resident_dit_paths=frozenset({"transformer"}),
)
def __init__(self):
super().__init__()
self.transformer = Transformer()
pipeline = Pipeline()
backend = DistributedLayerwiseOffloadBackend(
OffloadConfig(
strategy=OffloadStrategy.DISTRIBUTED_LAYER_WISE,
pin_cpu_memory=False,
dlo_use_allgather=False,
dlo_resident_layers=2,
),
torch.device("cpu"),
)
backend.enable(pipeline)
assert len(backend._resident_blocks) == 2
assert len(backend._all_hook_groups) == 1
assert len(backend._all_hook_groups[0]) == 2
assert backend.enabled
backend.disable()
class TestMmapValidation:
"""Tests for mmap loader validation: TP rejection, strict check, validate_loaded_weights."""
@@ -746,7 +1029,7 @@ class TestConfigValidation:
OffloadConfig.from_od_config(FakeODConfig())
def test_hsdp_without_allgather_allowed(self):
"""HSDP + DLO + no-AllGather should be allowed (full weights per rank)."""
"""HSDP + DLO + no-AllGather uses standard-loader rank-local weights."""
from vllm_omni.diffusion.offloader.base import OffloadConfig
class FakePC:
@@ -768,12 +1051,33 @@ class TestConfigValidation:
enable_layerwise_offload = False
enable_distributed_layerwise_offload = True
dlo_use_allgather = False # no AllGather → should be allowed
dlo_resident_layers = 20
pin_cpu_memory = True
parallel_config = FakePC()
model = "/fake/path"
config = OffloadConfig.from_od_config(FakeODConfig())
assert config.dp_size == 1 # forced to 1 when no AllGather
assert config.dlo_resident_layers == 20
def test_resident_layers_with_allgather_rejected(self):
class FakePC:
data_parallel_size = 2
use_hsdp = False
sequence_parallel_size = 1
class FakeODConfig:
enable_cpu_offload = False
enable_layerwise_offload = False
enable_distributed_layerwise_offload = True
dlo_use_allgather = True
dlo_resident_layers = 20
pin_cpu_memory = True
parallel_config = FakePC()
model = "/fake/path"
with pytest.raises(ValueError, match="requires --dlo-no-use-allgather"):
OffloadConfig.from_od_config(FakeODConfig())
def test_num_inference_steps_none_rejected(self):
"""DP multi-concurrency should reject None num_inference_steps."""
@@ -147,6 +147,130 @@ def test_get_npu_quant_method(mocker: MockerFixture, monkeypatch: pytest.MonkeyP
assert isinstance(method, UnquantizedLinearMethod)
def test_fused_layer_can_be_ignored_by_its_prefix(monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture):
"""A layer opts out of quantization under the prefix its module was built with.
MiniMax H3 stores qkv as one checkpoint tensor, so the fused name is the
only handle a user has on it.
"""
from vllm_omni.quantization.int8_config import NPUInt8OnlineLinearMethod
monkeypatch.setattr(current_omni_platform, "is_cuda", lambda: False)
monkeypatch.setattr(current_omni_platform, "is_npu", lambda: True)
config = build_quant_config("int8", ignored_layers=["blocks.0.attn.qkv_proj"])
layer = mocker.Mock(spec=LinearBase)
assert isinstance(config.get_quant_method(layer, "blocks.0.attn.qkv_proj"), UnquantizedLinearMethod)
assert isinstance(config.get_quant_method(layer, "blocks.1.attn.qkv_proj"), NPUInt8OnlineLinearMethod)
assert isinstance(config.get_quant_method(layer, "blocks.0.mlp.fc1"), NPUInt8OnlineLinearMethod)
class TestNPUQuantMatmulShapeFallback:
"""npu_quant_matmul rejects wide outputs, so those layers stay unquantized.
The limit applies to the per-rank shard, which is what the kernel sees. The
same layer can therefore be quantized at a higher TP degree.
"""
@pytest.fixture(autouse=True)
def _mock_tp(self, mocker):
# The fallback delegates to UnquantizedLinearMethod, which reads the TP
# group; stand one in so the shape gating can be tested without a
# distributed init.
mock_group = mocker.Mock()
mock_group.rank_in_group = 0
mocker.patch("vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size", return_value=1)
mocker.patch("vllm.model_executor.layers.linear.get_tensor_model_parallel_rank", return_value=0)
mocker.patch("vllm.distributed.parallel_state.get_tp_group", return_value=mock_group)
@pytest.fixture
def quant_config(self):
from vllm_omni.quantization.int8_config import DiffusionInt8Config
return DiffusionInt8Config(is_checkpoint_int8_serialized=False, activation_scheme="dynamic")
def create_weights(self, method, layer, output_partition_sizes):
# Narrow input dim: the fallback path allocates for real, and only the
# output dimension is under test.
method.create_weights(
layer,
input_size_per_partition=8,
output_partition_sizes=output_partition_sizes,
input_size=8,
output_size=sum(output_partition_sizes),
params_dtype=torch.bfloat16,
weight_loader=lambda param, loaded_weight, *args, **kwargs: None,
)
@pytest.mark.parametrize(
"method_name",
["NPUInt8LinearMethod", "NPUInt8OnlineLinearMethod"],
)
def test_over_wide_layer_falls_back_to_unquantized(self, quant_config, method_name):
import vllm_omni.quantization.int8_config as int8_config
method = getattr(int8_config, method_name)(quant_config)
layer = Module()
layer.quant_method = method
self.create_weights(method, layer, [int8_config.NPU_QUANT_MATMUL_MAX_OUT_FEATURES + 1])
assert isinstance(layer.quant_method, UnquantizedLinearMethod)
assert layer.weight.dtype == torch.bfloat16
def test_layer_within_the_limit_keeps_int8(self, quant_config):
from vllm_omni.quantization.int8_config import (
NPU_QUANT_MATMUL_MAX_OUT_FEATURES,
NPUInt8OnlineLinearMethod,
)
method = NPUInt8OnlineLinearMethod(quant_config)
layer = Module()
layer.quant_method = method
self.create_weights(method, layer, [NPU_QUANT_MATMUL_MAX_OUT_FEATURES])
assert layer.quant_method is method
def test_tensor_parallel_sharding_brings_a_wide_layer_back(self, quant_config):
from vllm_omni.quantization.int8_config import (
NPU_QUANT_MATMUL_MAX_OUT_FEATURES,
NPUInt8OnlineLinearMethod,
)
global_out_features = 4 * NPU_QUANT_MATMUL_MAX_OUT_FEATURES
method = NPUInt8OnlineLinearMethod(quant_config)
layer = Module()
layer.quant_method = method
self.create_weights(method, layer, [global_out_features // 4])
assert layer.quant_method is method
class TestOffloadAfterQuant:
def test_only_methods_advertising_the_capability_are_asked(self):
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.quantization.int8_config import DiffusionInt8Config, NPUInt8OnlineLinearMethod
class MetaDeviceOnlyMethod:
"""Stands in for upstream online FP8: lazy, but no offload hook."""
uses_meta_device = True
model = torch.nn.Sequential(Module(), Module())
lazy_int8 = NPUInt8OnlineLinearMethod(
DiffusionInt8Config(is_checkpoint_int8_serialized=False, activation_scheme="dynamic")
)
model[0].quant_method = lazy_int8
model[1].quant_method = MetaDeviceOnlyMethod()
marked = DiffusersPipelineLoader._request_offload_after_quant(model)
assert marked == 1
assert lazy_int8._offload_after_quant
class TestInt8LinearMethod:
@pytest.fixture
def mock_quant_config(self, mocker):
@@ -131,6 +131,11 @@ def test_qwen_image_edit_plus_sets_generic_multimodal_limit():
assert od_config.max_multimodal_image_inputs == QWEN_IMAGE_EDIT_PLUS_MAX_INPUT_IMAGES
def test_task_type_roundtrip():
od = _roundtrip_diffusion_config(model="x", task_type="model-defined-task")
assert od.task_type == "model-defined-task"
def test_additional_config_roundtrip():
additional_config = {"torchair_graph_config": {"enabled": True}}
od = _roundtrip_diffusion_config(model="x", additional_config=additional_config)
@@ -247,6 +247,56 @@ class TestRequestModeDispatch:
results = {ro.request_id: ro.result.error for ro in out.runner_outputs}
assert results == {request_id: f"result_for_{request_id}" for request_id in request_ids}
def test_dlo_dp_routes_multiple_requests_without_pipeline_batching(self):
executor, _, _ = _make_executor(num_gpus=2)
executor.od_config = SimpleNamespace(
step_execution=False,
parallel_config=SimpleNamespace(data_parallel_size=2),
enable_distributed_layerwise_offload=True,
dlo_use_allgather=True,
)
executor.execute_request = Mock(return_value="dlo-dp")
executor.collective_rpc = Mock()
scheduler_output = _make_sched_output("A", "B")
result = executor.execute_batch(scheduler_output)
assert result == "dlo-dp"
executor.execute_request.assert_called_once_with(scheduler_output)
executor.collective_rpc.assert_not_called()
def test_dlo_dp_multi_rank_reply_uses_synchronous_rpc_collection(self):
executor, req_q, res_q = _make_executor(num_gpus=2)
executor.od_config = SimpleNamespace(
step_execution=False,
parallel_config=SimpleNamespace(data_parallel_size=2),
)
executor._sync_result_buffer = res_q
def worker():
request = req_q.get(timeout=2)
assert "rpc_id" not in request
wave_id = request["wave_id"]
for dp_rank in range(2):
res_q.put(
{
"dp_rank": dp_rank,
"output": _tagged_output(str(dp_rank)),
"wave_id": wave_id,
}
)
thread = threading.Thread(target=worker, daemon=True)
thread.start()
results = executor.collective_rpc(
"execute_model",
unique_reply_rank=None,
exec_all_ranks=True,
)
thread.join(timeout=2)
assert [result.error for result in results] == ["0", "1"]
# ───────────────── concurrent collective RPC ─────────────────